{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "\n",
    "from sklearn.linear_model import LinearRegression\n",
    "from sklearn.metrics import mean_squared_error\n",
    "\n",
    "\n",
    "#用于计算feature字段的文本特征提取\n",
    "from sklearn.feature_extraction.text import  CountVectorizer\n",
    "#from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "\n",
    "#CountVectorizer为稀疏特征，特征编码结果存为稀疏矩阵xgboost处理更高效\n",
    "from scipy import sparse\n",
    "\n",
    "#对类别型特征进行编码\n",
    "from sklearn.preprocessing import LabelEncoder\n",
    "#from MeanEncoder import MeanEncoder\n",
    "\n",
    "#对地理位置通过聚类进行离散化\n",
    "from sklearn.cluster import KMeans\n",
    "from nltk.metrics import distance as distance\n",
    "\n",
    "#可视化\n",
    "from matplotlib import pyplot\n",
    "import seaborn as sns\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>bathrooms</th>\n",
       "      <th>bedrooms</th>\n",
       "      <th>building_id</th>\n",
       "      <th>created</th>\n",
       "      <th>description</th>\n",
       "      <th>display_address</th>\n",
       "      <th>features</th>\n",
       "      <th>interest_level</th>\n",
       "      <th>latitude</th>\n",
       "      <th>listing_id</th>\n",
       "      <th>longitude</th>\n",
       "      <th>manager_id</th>\n",
       "      <th>photos</th>\n",
       "      <th>price</th>\n",
       "      <th>street_address</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>1.5</td>\n",
       "      <td>3</td>\n",
       "      <td>53a5b119ba8f7b61d4e010512e0dfc85</td>\n",
       "      <td>2016-06-24 07:54:24</td>\n",
       "      <td>A Brand New 3 Bedroom 1.5 bath ApartmentEnjoy ...</td>\n",
       "      <td>Metropolitan Avenue</td>\n",
       "      <td>[]</td>\n",
       "      <td>medium</td>\n",
       "      <td>40.7145</td>\n",
       "      <td>7211212</td>\n",
       "      <td>-73.9425</td>\n",
       "      <td>5ba989232d0489da1b5f2c45f6688adc</td>\n",
       "      <td>[https://photos.renthop.com/2/7211212_1ed4542e...</td>\n",
       "      <td>3000</td>\n",
       "      <td>792 Metropolitan Avenue</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    bathrooms  bedrooms                       building_id  \\\n",
       "10        1.5         3  53a5b119ba8f7b61d4e010512e0dfc85   \n",
       "\n",
       "                created                                        description  \\\n",
       "10  2016-06-24 07:54:24  A Brand New 3 Bedroom 1.5 bath ApartmentEnjoy ...   \n",
       "\n",
       "        display_address features interest_level  latitude  listing_id  \\\n",
       "10  Metropolitan Avenue       []         medium   40.7145     7211212   \n",
       "\n",
       "    longitude                        manager_id  \\\n",
       "10   -73.9425  5ba989232d0489da1b5f2c45f6688adc   \n",
       "\n",
       "                                               photos  price  \\\n",
       "10  [https://photos.renthop.com/2/7211212_1ed4542e...   3000   \n",
       "\n",
       "             street_address  \n",
       "10  792 Metropolitan Avenue  "
      ]
     },
     "execution_count": 65,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#读数据\n",
    "train = pd.read_json(\"RentListingInquries_train.json\")\n",
    "test = pd.read_json(\"RentListingInquries_test.json\")\n",
    "train = train[:1000]\n",
    "test = test[:1000]\n",
    "train.head(1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "Int64Index: 1000 entries, 10 to 102241\n",
      "Data columns (total 15 columns):\n",
      "bathrooms          1000 non-null float64\n",
      "bedrooms           1000 non-null int64\n",
      "building_id        1000 non-null object\n",
      "created            1000 non-null object\n",
      "description        1000 non-null object\n",
      "display_address    1000 non-null object\n",
      "features           1000 non-null object\n",
      "interest_level     1000 non-null object\n",
      "latitude           1000 non-null float64\n",
      "listing_id         1000 non-null int64\n",
      "longitude          1000 non-null float64\n",
      "manager_id         1000 non-null object\n",
      "photos             1000 non-null object\n",
      "price              1000 non-null int64\n",
      "street_address     1000 non-null object\n",
      "dtypes: float64(3), int64(3), object(9)\n",
      "memory usage: 125.0+ KB\n"
     ]
    }
   ],
   "source": [
    "train.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "Int64Index: 1000 entries, 0 to 101495\n",
      "Data columns (total 14 columns):\n",
      "bathrooms          1000 non-null float64\n",
      "bedrooms           1000 non-null int64\n",
      "building_id        1000 non-null object\n",
      "created            1000 non-null object\n",
      "description        1000 non-null object\n",
      "display_address    1000 non-null object\n",
      "features           1000 non-null object\n",
      "latitude           1000 non-null float64\n",
      "listing_id         1000 non-null int64\n",
      "longitude          1000 non-null float64\n",
      "manager_id         1000 non-null object\n",
      "photos             1000 non-null object\n",
      "price              1000 non-null int64\n",
      "street_address     1000 non-null object\n",
      "dtypes: float64(3), int64(3), object(8)\n",
      "memory usage: 117.2+ KB\n"
     ]
    }
   ],
   "source": [
    "test.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# train['interest_level']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#将类别型的标签interest_level编码为数字\n",
    "y_map = {'low': 2, 'medium': 1, 'high': 0}\n",
    "train['interest_level'] = train['interest_level'].apply(lambda x: y_map[x])\n",
    "\n",
    "y_train = train['interest_level']\n",
    "train.drop(['listing_id', 'interest_level'], axis=1,inplace = True)\n",
    "\n",
    "test_id = test['listing_id']  #保留，生成提交文件时需要\n",
    "test.drop(['listing_id'], axis=1,inplace = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1    317\n",
       "2    312\n",
       "0    186\n",
       "3    138\n",
       "4     42\n",
       "5      5\n",
       "Name: bedrooms, dtype: int64"
      ]
     },
     "execution_count": 70,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train['bedrooms'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>bathrooms</th>\n",
       "      <th>bedrooms</th>\n",
       "      <th>building_id</th>\n",
       "      <th>created</th>\n",
       "      <th>description</th>\n",
       "      <th>display_address</th>\n",
       "      <th>features</th>\n",
       "      <th>latitude</th>\n",
       "      <th>longitude</th>\n",
       "      <th>manager_id</th>\n",
       "      <th>photos</th>\n",
       "      <th>price</th>\n",
       "      <th>street_address</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>1.5</td>\n",
       "      <td>3</td>\n",
       "      <td>53a5b119ba8f7b61d4e010512e0dfc85</td>\n",
       "      <td>2016-06-24 07:54:24</td>\n",
       "      <td>A Brand New 3 Bedroom 1.5 bath ApartmentEnjoy ...</td>\n",
       "      <td>Metropolitan Avenue</td>\n",
       "      <td>[]</td>\n",
       "      <td>40.7145</td>\n",
       "      <td>-73.9425</td>\n",
       "      <td>5ba989232d0489da1b5f2c45f6688adc</td>\n",
       "      <td>[https://photos.renthop.com/2/7211212_1ed4542e...</td>\n",
       "      <td>3000</td>\n",
       "      <td>792 Metropolitan Avenue</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    bathrooms  bedrooms                       building_id  \\\n",
       "10        1.5         3  53a5b119ba8f7b61d4e010512e0dfc85   \n",
       "\n",
       "                created                                        description  \\\n",
       "10  2016-06-24 07:54:24  A Brand New 3 Bedroom 1.5 bath ApartmentEnjoy ...   \n",
       "\n",
       "        display_address features  latitude  longitude  \\\n",
       "10  Metropolitan Avenue       []   40.7145   -73.9425   \n",
       "\n",
       "                          manager_id  \\\n",
       "10  5ba989232d0489da1b5f2c45f6688adc   \n",
       "\n",
       "                                               photos  price  \\\n",
       "10  [https://photos.renthop.com/2/7211212_1ed4542e...   3000   \n",
       "\n",
       "             street_address  \n",
       "10  792 Metropolitan Avenue  "
      ]
     },
     "execution_count": 71,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train.head(1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.axes._subplots.AxesSubplot at 0x79b62dd8>"
      ]
     },
     "execution_count": 72,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYgAAAELCAYAAADDZxFQAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4wLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvpW3flQAAEzlJREFUeJzt3X2wnnV95/H3ByK6VmlADiwmYeNq\nVmUfBDzjUtnpWLFdoVuT7TYVpkrKZiZ1hqpMd3Zld3Zwp+vO2GlXVmyXmUwRE8cqiKVkO6wtE2FZ\nqVADRkSiS8pScjaUHOTBB1Yd3O/+cf/OcEx+JHcw17lPct6vmXuu6/rd3+u6v+SQ88n1eKeqkCRp\nf8dNugFJ0uJkQEiSugwISVKXASFJ6jIgJEldBoQkqcuAkCR1DRYQSV6bZOe817eTXJ7k5CS3Jnmw\nTU9q9UlydZLdSe5Lcs5QvUmSDm2wgKiqb1bVWVV1FvBG4BngJuAKYHtVrQG2t2WAC4A17bUJuGao\n3iRJh7ZsgT7nfOCvquqvk6wF3tLGtwC3Ax8A1gJba3Rr911Jlic5vaoefb6NnnLKKbV69epBG5ek\nY80999zzeFVNHapuoQLiIuDTbf60uV/6VfVoklPb+Apgz7x1ZtrY8wbE6tWr2bFjxwDtStKxK8lf\nj1M3+EnqJCcA7wA+e6jSztgBD4pKsinJjiQ7Zmdnj0SLkqSOhbiK6QLg3qp6rC0/luR0gDbd18Zn\ngFXz1lsJ7N1/Y1W1uaqmq2p6auqQe0iSpBdoIQLiYp47vASwDdjQ5jcAN88bv6RdzXQu8PTBzj9I\nkoY16DmIJC8Ffh74jXnDHwZuSLIReARY38ZvAS4EdjO64unSIXuTJB3coAFRVc8Ar9hv7FuMrmra\nv7aAy4bsR5I0Pu+kliR1GRCSpC4DQpLUZUBIkroW6k5qSUvUeR87b9ItLAl3vvfOI75N9yAkSV0G\nhCSpy4CQJHUZEJKkLgNCktRlQEiSugwISVKXASFJ6jIgJEldBoQkqcuAkCR1GRCSpC4DQpLUZUBI\nkroMCElSlwEhSeoyICRJXYMGRJLlSW5M8o0ku5L8TJKTk9ya5ME2PanVJsnVSXYnuS/JOUP2Jkk6\nuKH3ID4KfL6qXge8AdgFXAFsr6o1wPa2DHABsKa9NgHXDNybJOkgBguIJCcCPwtcC1BVP6yqp4C1\nwJZWtgVY1+bXAltr5C5geZLTh+pPknRwQ+5B/F1gFrguyVeS/GGSnwJOq6pHAdr01Fa/Atgzb/2Z\nNiZJmoAhA2IZcA5wTVWdDXyP5w4n9aQzVgcUJZuS7EiyY3Z29sh0Kkk6wJABMQPMVNXdbflGRoHx\n2NyhozbdN69+1bz1VwJ7999oVW2uqumqmp6amhqseUla6gYLiKr6G2BPkte2ofOBB4BtwIY2tgG4\nuc1vAy5pVzOdCzw9dyhKkrTwlg28/fcCn0pyAvAQcCmjULohyUbgEWB9q70FuBDYDTzTaiVJEzJo\nQFTVTmC689b5ndoCLhuyH0nS+LyTWpLUZUBIkroMCElSlwEhSeoyICRJXQaEJKnLgJAkdRkQkqQu\nA0KS1GVASJK6DAhJUpcBIUnqMiAkSV0GhCSpy4CQJHUZEJKkLgNCktRlQEiSugwISVKXASFJ6jIg\nJEldBoQkqWvQgEjycJKvJdmZZEcbOznJrUkebNOT2niSXJ1kd5L7kpwzZG+SpINbiD2In6uqs6pq\nui1fAWyvqjXA9rYMcAGwpr02AdcsQG+SpOcxiUNMa4EtbX4LsG7e+NYauQtYnuT0CfQnSWL4gCjg\nz5Pck2RTGzutqh4FaNNT2/gKYM+8dWfamCRpApYNvP3zqmpvklOBW5N84yC16YzVAUWjoNkEcMYZ\nZxyZLiVJBxh0D6Kq9rbpPuAm4E3AY3OHjtp0XyufAVbNW30lsLezzc1VNV1V01NTU0O2L0lL2mAB\nkeSnkrx8bh74BeB+YBuwoZVtAG5u89uAS9rVTOcCT88dipIkLbwhDzGdBtyUZO5z/qiqPp/ky8AN\nSTYCjwDrW/0twIXAbuAZ4NIBe5MkHcJgAVFVDwFv6Ix/Czi/M17AZUP1I0k6PN5JLUnqMiAkSV0G\nhCSpy4CQJHUZEJKkLgNCktRlQEiSugwISVKXASFJ6jIgJEldBoQkqcuAkCR1GRCSpC4DQpLUZUBI\nkroMCElSlwEhSeoyICRJXQaEJKnLgJAkdRkQkqQuA0KS1DV4QCQ5PslXkvxpW35VkruTPJjk+iQn\ntPEXt+Xd7f3VQ/cmSXp+C7EH8X5g17zl3wGuqqo1wJPAxja+EXiyql4DXNXqJEkTMmhAJFkJ/CLw\nh205wFuBG1vJFmBdm1/blmnvn9/qJUkTMPQexH8B/g3w/9ryK4CnqurZtjwDrGjzK4A9AO39p1v9\nj0myKcmOJDtmZ2eH7F2SlrTBAiLJPwP2VdU984c7pTXGe88NVG2uqumqmp6amjoCnUqSepYNuO3z\ngHckuRB4CXAioz2K5UmWtb2ElcDeVj8DrAJmkiwDfhp4YsD+JEkHMdYeRJLt44zNV1X/tqpWVtVq\n4CLgC1X1a8BtwK+0sg3AzW1+W1umvf+FqjpgD0KStDAOugeR5CXAS4FTkpzEc4eBTgRe+QI/8wPA\nZ5J8CPgKcG0bvxb4ZJLdjPYcLnqB25ckHQGHOsT0G8DljMLgHp4LiG8DfzDuh1TV7cDtbf4h4E2d\nmu8D68fdpiRpWAcNiKr6KPDRJO+tqo8tUE+SpEVgrJPUVfWxJG8GVs9fp6q2DtSXJGnCxgqIJJ8E\nXg3sBH7UhgswICTpGDXuZa7TwJleVSRJS8e4N8rdD/ztIRuRJC0u4+5BnAI8kOQvgR/MDVbVOwbp\nSpI0ceMGxH8YsglJ0uIz7lVM/2PoRiRJi8u4VzF9h+cenHcC8CLge1V14lCNSZIma9w9iJfPX06y\njs7d0JKkY8cLetx3Vf0Joy/+kSQdo8Y9xPTL8xaPY3RfhPdESNIxbNyrmH5p3vyzwMOMviJUknSM\nGvccxKVDNyJJWlzG/cKglUluSrIvyWNJPpdk5dDNSZImZ9yT1Ncx+sa3VwIrgP/WxiRJx6hxA2Kq\nqq6rqmfb6xPA1IB9SZImbNyAeDzJu5Ic317vAr41ZGOSpMkaNyD+JfCrwN8AjwK/AnjiWpKOYeNe\n5vofgQ1V9SRAkpOB32MUHJKkY9C4exD/aC4cAKrqCeDsYVqSJC0G4wbEcUlOmltoexDj7n1Iko5C\n4/6S/8/AXyS5kdEjNn4V+E8HWyHJS4A7gBe3z7mxqj6Y5FXAZ4CTgXuBd1fVD5O8mNF3XL+R0Qnw\nd1bVw4f/nyRJOhLG2oOoqq3AvwAeA2aBX66qTx5itR8Ab62qNwBnAW9Pci7wO8BVVbUGeBLY2Oo3\nAk9W1WuAq1qdJGlCxn6aa1U9UFW/X1Ufq6oHxqivqvpuW3xRexWjp8De2Ma3AOva/Nq2THv//CQZ\ntz9J0pH1gh73Pa52z8ROYB9wK/BXwFNV9WwrmWF0ZzZtugegvf808IrONjcl2ZFkx+zs7JDtS9KS\nNmhAVNWPquosYCWjLxh6fa+sTXt7Cwc8UryqNlfVdFVNT015M7ckDWXQgJhTVU8BtwPnAsuTzJ0c\nXwnsbfMzwCqA9v5PA08sRH+SpAMNFhBJppIsb/N/C3gbsAu4jdGd2AAbgJvb/La2THv/C1XllxJJ\n0oQMeS/D6cCWJMczCqIbqupPkzwAfCbJh4CvANe2+muBTybZzWjP4aIBe5MkHcJgAVFV99G527qq\nHmJ0PmL/8e8D64fqR5J0eBbkHIQk6ehjQEiSugwISVKXASFJ6jIgJEldBoQkqcuAkCR1GRCSpC4D\nQpLU5deG6qjwyG//w0m3cMw748qvTboFLTLuQUiSugwISVKXASFJ6jIgJEldBoQkqcuAkCR1GRCS\npC4DQpLUZUBIkroMCElSlwEhSeoaLCCSrEpyW5JdSb6e5P1t/OQktyZ5sE1PauNJcnWS3UnuS3LO\nUL1Jkg5tyD2IZ4F/VVWvB84FLktyJnAFsL2q1gDb2zLABcCa9toEXDNgb5KkQxgsIKrq0aq6t81/\nB9gFrADWAlta2RZgXZtfC2ytkbuA5UlOH6o/SdLBLcg5iCSrgbOBu4HTqupRGIUIcGorWwHsmbfa\nTBuTJE3A4AGR5GXA54DLq+rbByvtjFVne5uS7EiyY3Z29ki1KUnaz6ABkeRFjMLhU1X1x234sblD\nR226r43PAKvmrb4S2Lv/Nqtqc1VNV9X01NTUcM1L0hI35FVMAa4FdlXVR+a9tQ3Y0OY3ADfPG7+k\nXc10LvD03KEoSdLCG/IrR88D3g18LcnONvbvgA8DNyTZCDwCrG/v3QJcCOwGngEuHbA3SdIhDBYQ\nVfVF+ucVAM7v1Bdw2VD9SJIOj3dSS5K6DAhJUpcBIUnqMiAkSV0GhCSpy4CQJHUZEJKkLgNCktRl\nQEiSugwISVKXASFJ6jIgJEldBoQkqcuAkCR1GRCSpC4DQpLUZUBIkroMCElSlwEhSeoyICRJXQaE\nJKnLgJAkdQ0WEEk+nmRfkvvnjZ2c5NYkD7bpSW08Sa5OsjvJfUnOGaovSdJ4htyD+ATw9v3GrgC2\nV9UaYHtbBrgAWNNem4BrBuxLkjSGwQKiqu4AnthveC2wpc1vAdbNG99aI3cBy5OcPlRvkqRDW+hz\nEKdV1aMAbXpqG18B7JlXN9PGJEkTslhOUqczVt3CZFOSHUl2zM7ODtyWJC1dCx0Qj80dOmrTfW18\nBlg1r24lsLe3garaXFXTVTU9NTU1aLOStJQtdEBsAza0+Q3AzfPGL2lXM50LPD13KEqSNBnLhtpw\nkk8DbwFOSTIDfBD4MHBDko3AI8D6Vn4LcCGwG3gGuHSoviRJ4xksIKrq4ud56/xObQGXDdWLJOnw\nLZaT1JKkRcaAkCR1DXaIabF547/eOukWloR7fveSSbcg6QhxD0KS1GVASJK6DAhJUpcBIUnqMiAk\nSV0GhCSpy4CQJHUZEJKkLgNCktRlQEiSugwISVKXASFJ6jIgJEldBoQkqcuAkCR1GRCSpC4DQpLU\nZUBIkroMCElS16IKiCRvT/LNJLuTXDHpfiRpKVs0AZHkeOAPgAuAM4GLk5w52a4kaelaNAEBvAnY\nXVUPVdUPgc8AayfckyQtWYspIFYAe+Ytz7QxSdIELJt0A/OkM1YHFCWbgE1t8btJvjloV5N1CvD4\npJs4HPm9DZNuYbE46n52fLD3V3DJOup+fnnfYf38/s44RYspIGaAVfOWVwJ79y+qqs3A5oVqapKS\n7Kiq6Un3ocPnz+7o5s9vZDEdYvoysCbJq5KcAFwEbJtwT5K0ZC2aPYiqejbJbwJ/BhwPfLyqvj7h\ntiRpyVo0AQFQVbcAt0y6j0VkSRxKO0b5szu6+fMDUnXAeWBJkhbVOQhJ0iJiQCxCPnLk6JXk40n2\nJbl/0r3o8CVZleS2JLuSfD3J+yfd0yR5iGmRaY8c+V/AzzO69PfLwMVV9cBEG9NYkvws8F1ga1X9\ng0n3o8OT5HTg9Kq6N8nLgXuAdUv17597EIuPjxw5ilXVHcATk+5DL0xVPVpV97b57wC7WMJPdDAg\nFh8fOSItAklWA2cDd0+2k8kxIBafsR45Imk4SV4GfA64vKq+Pel+JsWAWHzGeuSIpGEkeRGjcPhU\nVf3xpPuZJANi8fGRI9KEJAlwLbCrqj4y6X4mzYBYZKrqWWDukSO7gBt85MjRI8mngS8Br00yk2Tj\npHvSYTkPeDfw1iQ72+vCSTc1KV7mKknqcg9CktRlQEiSugwISVKXASFJ6jIgJEldBoQkqcuA0DEp\nyV+MUXN5kpcO3MdZh7qOPsmvJ/n9I/y5R3ybWnoMCB2TqurNY5RdDhxWQLTHsR+Os4Ale6OVjm4G\nhI5JSb7bpm9JcnuSG5N8I8mnMvI+4JXAbUlua7W/kORLSe5N8tn2wDaSPJzkyiRfBNYneXWSzye5\nJ8n/TPK6Vrc+yf1JvprkjvaolN8G3tnuyH3nGH1PJflcki+313lJjms9LJ9XtzvJab36I/6HqSVr\n2aQbkBbA2cDfZ/TQwzuB86rq6iS/BfxcVT2e5BTg3wNvq6rvJfkA8FuMfsEDfL+q/glAku3Ae6rq\nwST/GPivwFuBK4F/WlX/J8nyqvphkiuB6ar6zTF7/ShwVVV9MckZwJ9V1euT3Az8c+C69pkPV9Vj\nSf5o/3rg9T/hn5cEGBBaGv6yqmYAkuwEVgNf3K/mXOBM4M7R89o4gdEzleZc39Z/GfBm4LOtDuDF\nbXon8IkkNwAv9CmgbwPOnLftE9s3m13PKICuY/QAx+sPUS/9xAwILQU/mDf/I/r/3we4taoufp5t\nfK9NjwOeqqqz9i+oqve0f93/IrAzyQE1YzgO+Jmq+r8/1lzyJeA1SaaAdcCHDlH/Aj5a+nGeg9BS\n9h1g7l/bdwHnJXkNQJKXJvl7+6/QvjzmfydZ3+qS5A1t/tVVdXdVXQk8zuh7PeZ/xjj+nNHTfGnb\nPKt9bgE3AR9h9Cjqbx2sXjoSDAgtZZuB/57ktqqaBX4d+HSS+xgFxuueZ71fAzYm+SrwdZ77zvDf\nTfK1JPcDdwBfBW5jdAhorJPUwPuA6ST3JXkAeM+8964H3sVzh5cOVS/9RHzctySpyz0ISVKXJ6ml\nBZLkUuD9+w3fWVWXTaIf6VA8xCRJ6vIQkySpy4CQJHUZEJKkLgNCktRlQEiSuv4/GdQX8NcjtFQA\nAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x4bc7cfd0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "sns.countplot(y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [],
   "source": [
    "# train.info()\n",
    "# train[]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def remove_noise(df):\n",
    "#remove some noise\n",
    "    df= df[df.price < 10000]\n",
    "\n",
    "    df.loc[df[\"bathrooms\"] == 112, \"bathrooms\"] = 1.5\n",
    "    df.loc[df[\"bathrooms\"] == 10, \"bathrooms\"] = 1\n",
    "    df.loc[df[\"bathrooms\"] == 20, \"bathrooms\"] = 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#构造新特征\n",
    "#price_bathrooms：单位bathroom的价格\n",
    "#price_bedrooms：单位bedroom的价格\n",
    "def create_price_room(df):\n",
    "    df['price_bathrooms'] =  (df[\"price\"])/ (df[\"bathrooms\"] +1.0)\n",
    "    df['price_bedrooms'] =  (df[\"price\"])/ (df[\"bedrooms\"] +1.0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#构造新特征\n",
    "#room_diff：bathroom房间数 - bedroom房间数\n",
    "#room_num：bathroom房间数 + bedroom房间数\n",
    "def create_room_diff_sum(df):\n",
    "    df[\"room_diff\"] = df[\"bathrooms\"] - df[\"bedrooms\"]\n",
    "    df[\"room_num\"] = df[\"bedrooms\"] + df[\"bathrooms\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def procdess_created_date(df):\n",
    "    df['Date'] = pd.to_datetime(df['created'])\n",
    "    df['Year'] = df['Date'].dt.year\n",
    "    df['Month'] = df['Date'].dt.month\n",
    "    df['Day'] = df['Date'].dt.day\n",
    "    df['Wday'] = df['Date'].dt.dayofweek\n",
    "    df['Yday'] = df['Date'].dt.dayofyear\n",
    "    df['hour'] = df['Date'].dt.hour\n",
    "\n",
    "    df.drop(['Date', 'created'], axis=1,inplace = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#简单丢弃，也可以参照feature特征处理方式\n",
    "def procdess_description(df):\n",
    "    df.drop(['description'], axis=1,inplace = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def procdess_manager_id(df):\n",
    "    managers_count = df['manager_id'].value_counts()\n",
    "\n",
    "    df['top_10_manager'] = df['manager_id'].apply(lambda x: 1 if x in managers_count.index.values[\n",
    "        managers_count.values >= np.percentile(managers_count.values, 90)] else 0)\n",
    "    df['top_25_manager'] = df['manager_id'].apply(lambda x: 1 if x in managers_count.index.values[\n",
    "        managers_count.values >= np.percentile(managers_count.values, 75)] else 0)\n",
    "    df['top_5_manager'] = df['manager_id'].apply(lambda x: 1 if x in managers_count.index.values[\n",
    "        managers_count.values >= np.percentile(managers_count.values, 95)] else 0)\n",
    "    df['top_50_manager'] = df['manager_id'].apply(lambda x: 1 if x in managers_count.index.values[\n",
    "        managers_count.values >= np.percentile(managers_count.values, 50)] else 0)\n",
    "    df['top_1_manager'] = df['manager_id'].apply(lambda x: 1 if x in managers_count.index.values[\n",
    "        managers_count.values >= np.percentile(managers_count.values, 99)] else 0)\n",
    "    df['top_2_manager'] = df['manager_id'].apply(lambda x: 1 if x in managers_count.index.values[\n",
    "        managers_count.values >= np.percentile(managers_count.values, 98)] else 0)\n",
    "    df['top_15_manager'] = df['manager_id'].apply(lambda x: 1 if x in managers_count.index.values[\n",
    "        managers_count.values >= np.percentile(managers_count.values, 85)] else 0)\n",
    "    df['top_20_manager'] = df['manager_id'].apply(lambda x: 1 if x in managers_count.index.values[\n",
    "        managers_count.values >= np.percentile(managers_count.values, 80)] else 0)\n",
    "    df['top_30_manager'] = df['manager_id'].apply(lambda x: 1 if x in managers_count.index.values[\n",
    "        managers_count.values >= np.percentile(managers_count.values, 70)] else 0)\n",
    "    \n",
    "    df.drop(['manager_id'], axis=1,inplace = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def procdess_building_id(df):\n",
    "    df.drop(['building_id'], axis=1,inplace = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def procdess_photos(df):\n",
    "    #df['photos_count'] = df['photos'].apply(lambda x: len(x))\n",
    "    df.drop(['photos'], axis=1, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def procdess_location_train(df):   \n",
    "    train_location = df.loc[:,[ 'latitude', 'longitude']]\n",
    "    \n",
    "     # Clustering\n",
    "    kmeans_cluster = KMeans(n_clusters=20)\n",
    "    res = kmeans_cluster.fit(train_location)\n",
    "    res = kmeans_cluster.predict(train_location)\n",
    "\n",
    "    df['cenroid'] = res\n",
    "\n",
    "    # L1 distance\n",
    "    center = [ train_location['latitude'].mean(), train_location['longitude'].mean()]\n",
    "    df['distance'] = abs(df['latitude'] - center[0]) + abs(df['longitude'] - center[1])\n",
    "    \n",
    "    #原始特征也可以考虑保留，此处简单丢弃\n",
    "    df.drop(['latitude', 'longitude'], axis=1, inplace=True)\n",
    "    \n",
    "    return kmeans_cluster,center"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def procdess_location_test(df, kmeans_cluster, center):   \n",
    "    test_location = df.loc[:,[ 'latitude', 'longitude']]\n",
    "    \n",
    "     # Clustering\n",
    "    res = kmeans_cluster.predict(test_location)\n",
    "\n",
    "    df['cenroid'] = res\n",
    "\n",
    "    # L1 distance\n",
    "    df['distance'] = abs(df['latitude'] - center[0]) + abs(df['longitude'] - center[1])\n",
    "    df.drop(['latitude', 'longitude'], axis=1, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def procdess_display_address_train_test(df_train, y_train, df_test):\n",
    "    n_train_samples = len(df_train.index)    \n",
    "    df_train_test = pd.concat((df_train, df_test), axis=0)\n",
    "\n",
    "    lb = LabelEncoder()\n",
    "    lb.fit(list(df_train_test['display_address'].values))\n",
    "    df_train_test ['display_address'] = lb.transform(list(df_train_test['display_address'].values))\n",
    "    \n",
    "    #import pdb\n",
    "    #pdb.set_trace()\n",
    "    me = MeanEncoder(['display_address'], target_type='classification')\n",
    "    \n",
    "    #下列函数并没有对test部分进行编码，因为fit还是根据y_train来训练的\n",
    "    #放弃该函数，直接丢弃算了，文本特征编码需要额外的功力\n",
    "    df_train_test = me.fit_transform(df_train_test, y_train)\n",
    "\n",
    "    df_train_test.drop(['display_address'], axis=1,inplace = True)\n",
    "    \n",
    "    df_train = df_train_test.iloc[:n_train_samples, :]\n",
    "    df_test = df_train_test.iloc[n_train_samples:, :]\n",
    "    \n",
    "    return df_train, df_test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def procdess_display_address(df):\n",
    "    df.drop(['display_address'], axis=1,inplace = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# 和display_address信息冗余，去掉\n",
    "def procdess_street_address(df):\n",
    "    df = df.drop(['street_address'], axis=1,inplace = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def procdess_features_train_test(df_train, df_test):\n",
    "    n_train_samples = len(df_train.index)\n",
    "    \n",
    "    df_train_test = pd.concat((df_train, df_test), axis=0)\n",
    "    df_train_test['features2'] = df_train_test['features']\n",
    "    df_train_test['features2'] = df_train_test['features2'].apply(lambda x: ' '.join(x))\n",
    "\n",
    "    c_vect = CountVectorizer(stop_words='english', max_features=200, ngram_range=(1, 1), decode_error='ignore')\n",
    "    c_vect_sparse = c_vect.fit_transform(df_train_test['features2'])\n",
    "    c_vect_sparse_cols = c_vect.get_feature_names()\n",
    "\n",
    "    df_train.drop(['features'], axis=1, inplace=True)\n",
    "    df_test.drop(['features'], axis=1, inplace=True)\n",
    "    \n",
    "    #hstack作为特征处理的最后一部，先将其他所有特征都转换成数值型特征才能处理,稀疏表示\n",
    "    df_train_sparse = sparse.hstack([df_train, c_vect_sparse[:n_train_samples,:]]).tocsr()\n",
    "    df_test_sparse = sparse.hstack([df_test, c_vect_sparse[n_train_samples:,:]]).tocsr()\n",
    "    \n",
    "    #常规datafrmae\n",
    "    tmp = pd.DataFrame(c_vect_sparse.toarray()[:n_train_samples,:],columns = c_vect_sparse_cols, index=df_train.index)\n",
    "    df_train = pd.concat([df_train, tmp], axis=1)\n",
    "    \n",
    "    tmp = pd.DataFrame(c_vect_sparse.toarray()[n_train_samples:,:],columns = c_vect_sparse_cols, index=df_test.index)\n",
    "    df_test = pd.concat([df_test, tmp], axis=1)\n",
    "    \n",
    "    #df_test = pd.concat([df_test, tmp[n_train_samples:,:]], axis=1)\n",
    "  \n",
    "    return df_train_sparse,df_test_sparse,df_train, df_test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def procdess_features_test(df, c_vect):\n",
    "    df['features2'] = df['features']\n",
    "    df['features2'] = df['features2'].apply(lambda x: ' '.join(x))\n",
    "\n",
    "    c_vect_sparse = c_vect.transform(df['features2'])\n",
    "    c_vect_sparse_cols = c_vect.get_feature_names()\n",
    "\n",
    "    df.drop(['features', 'features2'], axis=1, inplace=True)\n",
    "    \n",
    "    #hstack作为特征处理的最后一部，先将其他所有特征都转换成数值型特征才能处理\n",
    "    df_sparse = sparse.hstack([df, c_vect_sparse]).tocsr()\n",
    "    \n",
    "    tmp = pd.DataFrame(c_vect_sparse.toarray(),columns = c_vect_sparse_cols, index=df.index)\n",
    "    df = pd.concat([df, tmp], axis=1)\n",
    "    \n",
    "    return df_sparse, df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {},
   "outputs": [],
   "source": [
    "remove_noise(train)\n",
    "\n",
    "create_price_room(train)\n",
    "create_room_diff_sum(train)\n",
    "\n",
    "procdess_created_date(train)\n",
    "\n",
    "procdess_description(train)\n",
    "\n",
    "procdess_manager_id(train)\n",
    "\n",
    "procdess_building_id(train)\n",
    "procdess_photos(train)\n",
    "\n",
    "#kmeans_cluster,center = procdess_location_train(train)\n",
    "procdess_street_address(train)\n",
    "procdess_display_address(train)\n",
    "#测试集中可能出现新的特征值，所以训练和测试集一起做\n",
    "#lb, me, train = procdess_display_address_train(train, y_train)\n",
    "#X_train_sparse,X_test_sparse,train,test = procdess_features_train_test(train,test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "remove_noise(test)\n",
    "\n",
    "create_price_room(test)\n",
    "create_room_diff_sum(test)\n",
    "\n",
    "procdess_created_date(test)\n",
    "\n",
    "procdess_description(test)\n",
    "\n",
    "procdess_manager_id(test)\n",
    "\n",
    "procdess_building_id(test)\n",
    "procdess_photos(test)\n",
    "\n",
    "#还是直接用经纬度\n",
    "#procdess_location_test(test, kmeans_cluster, center)\n",
    "\n",
    "procdess_street_address(test)\n",
    "procdess_display_address(test)\n",
    "#测试数据出现了训练数据中没有出现的词语，报错，可以训练数据和测试数据一起训练CountVectorizer\n",
    "#test = procdess_display_address_test(test, lb, me )\n",
    "#X_test_sparse,test = procdess_features_test(test, c_vect)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "X_train_sparse,X_test_sparse,train,test = procdess_features_train_test(train,test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# #存为csv格式方便用excel查看(属性名字有重复，features得到的词语中也有bathrooms和bedrooms)\n",
    "# train = pd.concat([train, y_train], axis=1)\n",
    "# test= pd.concat([test_id,test],axis = 1)\n",
    "# train.to_csv('RentListingInquries_FE_train.csv', index=False)\n",
    "# test.to_csv('RentListingInquries_FE_test.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>bathrooms</th>\n",
       "      <th>bedrooms</th>\n",
       "      <th>latitude</th>\n",
       "      <th>longitude</th>\n",
       "      <th>price</th>\n",
       "      <th>price_bathrooms</th>\n",
       "      <th>price_bedrooms</th>\n",
       "      <th>room_diff</th>\n",
       "      <th>room_num</th>\n",
       "      <th>Year</th>\n",
       "      <th>...</th>\n",
       "      <th>walk</th>\n",
       "      <th>walls</th>\n",
       "      <th>war</th>\n",
       "      <th>washer</th>\n",
       "      <th>wheelchair</th>\n",
       "      <th>wic</th>\n",
       "      <th>wifi</th>\n",
       "      <th>windows</th>\n",
       "      <th>work</th>\n",
       "      <th>yoga</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>1.5</td>\n",
       "      <td>3</td>\n",
       "      <td>40.7145</td>\n",
       "      <td>-73.9425</td>\n",
       "      <td>3000</td>\n",
       "      <td>1200.0</td>\n",
       "      <td>750.0</td>\n",
       "      <td>-1.5</td>\n",
       "      <td>4.5</td>\n",
       "      <td>2016</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1 rows × 224 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "    bathrooms  bedrooms  latitude  longitude  price  price_bathrooms  \\\n",
       "10        1.5         3   40.7145   -73.9425   3000           1200.0   \n",
       "\n",
       "    price_bedrooms  room_diff  room_num  Year  ...   walk  walls  war  washer  \\\n",
       "10           750.0       -1.5       4.5  2016  ...      0      0    0       0   \n",
       "\n",
       "    wheelchair  wic  wifi  windows  work  yoga  \n",
       "10           0    0     0        0     0     0  \n",
       "\n",
       "[1 rows x 224 columns]"
      ]
     },
     "execution_count": 95,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train.head(1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.preprocessing import StandardScaler\n",
    "\n",
    "ss_X = StandardScaler()\n",
    "X_train = ss_X.fit_transform(train)\n",
    "X_test = ss_X.fit_transform(test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.linear_model import LogisticRegression\n",
    "lr = LogisticRegression()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "logloss of each fold is:  [ 0.90989768  0.97985732  0.74476877  0.85863274  0.87071757]\n",
      "cv logloss is: 0.872774816012\n"
     ]
    }
   ],
   "source": [
    "from sklearn.cross_validation import cross_val_score\n",
    "loss = cross_val_score(lr,X_train,y_train,cv=5,scoring='neg_log_loss')\n",
    "print 'logloss of each fold is: ',-loss\n",
    "print'cv logloss is:', -loss.mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "GridSearchCV(cv=5, error_score='raise',\n",
       "       estimator=LogisticRegression(C=1.0, class_weight=None, dual=False, fit_intercept=True,\n",
       "          intercept_scaling=1, max_iter=100, multi_class='ovr', n_jobs=1,\n",
       "          penalty='l2', random_state=None, solver='liblinear', tol=0.0001,\n",
       "          verbose=0, warm_start=False),\n",
       "       fit_params=None, iid=True, n_jobs=1,\n",
       "       param_grid={'penalty': ['l1', 'l2'], 'C': [0.001, 0.01, 0.1, 1, 10, 100, 1000]},\n",
       "       pre_dispatch='2*n_jobs', refit=True, return_train_score='warn',\n",
       "       scoring='neg_log_loss', verbose=0)"
      ]
     },
     "execution_count": 101,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.model_selection import GridSearchCV\n",
    "\n",
    "penaltys = ['l1','l2']\n",
    "Cs = [0.001,0.01,0.1,1,10,100,1000]\n",
    "tuned_parameters = dict(penalty = penaltys,C = Cs)\n",
    "\n",
    "lr_penalty = LogisticRegression()\n",
    "grid = GridSearchCV(lr_penalty,tuned_parameters,cv=5,scoring='neg_log_loss')\n",
    "grid.fit(X_train,y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "D:\\train\\Anaconda2\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('mean_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "D:\\train\\Anaconda2\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('split0_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "D:\\train\\Anaconda2\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('split1_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "D:\\train\\Anaconda2\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('split2_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "D:\\train\\Anaconda2\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('split3_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "D:\\train\\Anaconda2\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('split4_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "D:\\train\\Anaconda2\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('std_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'mean_fit_time': array([  0.02340002,   0.05499997,   0.02680006,   0.09719996,\n",
       "          0.0796    ,   0.18019996,   0.50300007,   0.41919999,\n",
       "          2.19759994,   0.8658    ,   5.07540002,   1.26599998,\n",
       "         10.16919999,   1.3816    ]),\n",
       " 'mean_score_time': array([ 0.00339994,  0.00180006,  0.00199995,  0.00239997,  0.00199995,\n",
       "         0.00080004,  0.00199995,  0.00179996,  0.00360003,  0.00040002,\n",
       "         0.00040002,  0.00740004,  0.00040002,  0.00379996]),\n",
       " 'mean_test_score': array([-1.09861229, -1.01501946, -0.89257478, -0.84666185, -0.7272205 ,\n",
       "        -0.80386038, -0.78277564, -0.87286526, -0.90608174, -0.96430663,\n",
       "        -1.03777928, -1.06039944, -1.25020592, -1.16356497]),\n",
       " 'mean_train_score': array([-1.09861229, -0.98344339, -0.89242222, -0.74557268, -0.66256761,\n",
       "        -0.60429311, -0.57646449, -0.57143521, -0.5670831 , -0.56677169,\n",
       "        -0.56622181, -0.56619552, -0.5661421 , -0.56613503]),\n",
       " 'param_C': masked_array(data = [0.001 0.001 0.01 0.01 0.1 0.1 1 1 10 10 100 100 1000 1000],\n",
       "              mask = [False False False False False False False False False False False False\n",
       "  False False],\n",
       "        fill_value = ?),\n",
       " 'param_penalty': masked_array(data = ['l1' 'l2' 'l1' 'l2' 'l1' 'l2' 'l1' 'l2' 'l1' 'l2' 'l1' 'l2' 'l1' 'l2'],\n",
       "              mask = [False False False False False False False False False False False False\n",
       "  False False],\n",
       "        fill_value = ?),\n",
       " 'params': [{'C': 0.001, 'penalty': 'l1'},\n",
       "  {'C': 0.001, 'penalty': 'l2'},\n",
       "  {'C': 0.01, 'penalty': 'l1'},\n",
       "  {'C': 0.01, 'penalty': 'l2'},\n",
       "  {'C': 0.1, 'penalty': 'l1'},\n",
       "  {'C': 0.1, 'penalty': 'l2'},\n",
       "  {'C': 1, 'penalty': 'l1'},\n",
       "  {'C': 1, 'penalty': 'l2'},\n",
       "  {'C': 10, 'penalty': 'l1'},\n",
       "  {'C': 10, 'penalty': 'l2'},\n",
       "  {'C': 100, 'penalty': 'l1'},\n",
       "  {'C': 100, 'penalty': 'l2'},\n",
       "  {'C': 1000, 'penalty': 'l1'},\n",
       "  {'C': 1000, 'penalty': 'l2'}],\n",
       " 'rank_test_score': array([12,  9,  6,  4,  1,  3,  2,  5,  7,  8, 10, 11, 14, 13]),\n",
       " 'split0_test_score': array([-1.09861229, -1.01503239, -0.89343198, -0.84914496, -0.7474676 ,\n",
       "        -0.82719961, -0.81065549, -0.90989768, -0.95721168, -0.99527663,\n",
       "        -1.06964286, -1.07293704, -1.18807424, -1.16172702]),\n",
       " 'split0_train_score': array([-1.09861229, -0.98297482, -0.8906457 , -0.74228357, -0.65472319,\n",
       "        -0.59279291, -0.55907151, -0.55371652, -0.54770885, -0.54747241,\n",
       "        -0.54667088, -0.54663836, -0.54657501, -0.54656274]),\n",
       " 'split1_test_score': array([-1.09861229, -1.02222816, -0.89394212, -0.87387086, -0.74630808,\n",
       "        -0.85484218, -0.83864368, -0.97985732, -1.03838005, -1.1432642 ,\n",
       "        -1.29422153, -1.31375658, -1.8029689 , -1.50318729]),\n",
       " 'split1_train_score': array([-1.09861229, -0.98175238, -0.89197715, -0.7441094 , -0.65722034,\n",
       "        -0.60434466, -0.5753645 , -0.5715518 , -0.56723828, -0.56701071,\n",
       "        -0.56642652, -0.56641287, -0.56633975, -0.56633891]),\n",
       " 'split2_test_score': array([-1.09861229, -1.01059142, -0.894164  , -0.81222479, -0.70420512,\n",
       "        -0.72355301, -0.70859197, -0.74476877, -0.76266862, -0.7919754 ,\n",
       "        -0.82358544, -0.84590707, -0.88162923, -0.90071723]),\n",
       " 'split2_train_score': array([-1.09861229, -0.98402379, -0.89213863, -0.75061726, -0.6676324 ,\n",
       "        -0.61360145, -0.58993621, -0.5846062 , -0.58102546, -0.58064054,\n",
       "        -0.58025578, -0.58021841, -0.58019085, -0.58018091]),\n",
       " 'split3_test_score': array([-1.09861229, -1.01327975, -0.88972643, -0.84677739, -0.7292185 ,\n",
       "        -0.80733607, -0.77058198, -0.85863274, -0.84579619, -0.91977767,\n",
       "        -0.92053078, -0.9882147 , -1.03214338, -1.06476858]),\n",
       " 'split3_train_score': array([-1.09861229, -0.98305049, -0.89265779, -0.74258886, -0.66290848,\n",
       "        -0.60090663, -0.57421258, -0.56882796, -0.56515478, -0.56471272,\n",
       "        -0.56426831, -0.56423633, -0.56418659, -0.56417825]),\n",
       " 'split4_test_score': array([-1.09861229, -1.01395143, -0.89158157, -0.85129016, -0.70861768,\n",
       "        -0.80616653, -0.78507681, -0.87071757, -0.92563721, -0.97073906,\n",
       "        -1.08022313, -1.08079751, -1.34622498, -1.1870666 ]),\n",
       " 'split4_train_score': array([-1.09861229, -0.98541545, -0.89469184, -0.74826429, -0.67035365,\n",
       "        -0.60981989, -0.58373766, -0.57847358, -0.57428814, -0.57402208,\n",
       "        -0.57348756, -0.57347163, -0.5734183 , -0.57341433]),\n",
       " 'std_fit_time': array([  5.00398448e-03,   6.06630086e-03,   7.48366451e-04,\n",
       "          1.93911829e-03,   1.07814687e-02,   1.00677530e-02,\n",
       "          1.18350399e-01,   1.25124267e-02,   5.90991268e-01,\n",
       "          2.41611078e-02,   4.21987961e-01,   5.69805988e-02,\n",
       "          8.80655526e-01,   1.12120604e-01]),\n",
       " 'std_score_time': array([  1.85475381e-03,   4.00066376e-04,   1.16800773e-07,\n",
       "          4.89940316e-04,   6.32485093e-04,   9.79841683e-04,\n",
       "          6.32485093e-04,   4.00018706e-04,   6.24820782e-03,\n",
       "          8.00037384e-04,   8.00037384e-04,   7.08801695e-03,\n",
       "          8.00037384e-04,   6.20968822e-03]),\n",
       " 'std_test_score': array([ 0.        ,  0.00389152,  0.00168472,  0.01976763,  0.01820107,\n",
       "         0.04385407,  0.04373265,  0.07674278,  0.09455925,  0.11375999,\n",
       "         0.16008624,  0.15228536,  0.31679069,  0.19733076]),\n",
       " 'std_train_score': array([  1.40433339e-16,   1.22142019e-03,   1.31462364e-03,\n",
       "          3.30318509e-03,   5.94170148e-03,   7.22379350e-03,\n",
       "          1.04228887e-02,   1.04285855e-02,   1.11839724e-02,\n",
       "          1.11505331e-02,   1.12768831e-02,   1.12778683e-02,\n",
       "          1.12875557e-02,   1.12891119e-02])}"
      ]
     },
     "execution_count": 102,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "grid.cv_results_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.727220495547\n",
      "{'penalty': 'l1', 'C': 0.1}\n"
     ]
    }
   ],
   "source": [
    "print(-grid.best_score_)\n",
    "print(grid.best_params_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAY4AAAEKCAYAAAAFJbKyAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4wLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvpW3flQAAIABJREFUeJzs3Xd8k1XbwPHfSZruvUsHXZRZWmZZ\nsmfZU1SUR0GQoYL7FRXHo6IiQ4YyBPERBWUICohs2RtkjxbooEAZXUBpm5z3j9QKWiClTdOW8/18\nKknuc9/3FaG5craQUqIoiqIoptJYOgBFURSlfFGJQ1EURSkSlTgURVGUIlGJQ1EURSkSlTgURVGU\nIlGJQ1EURSkSlTgURVGUIlGJQ1EURSkSlTgURVGUIrGydADm4OnpKYODgy0dhqIoSrmxd+/ey1JK\nL1PKVsjEERwczJ49eywdhqIoSrkhhDhnalnVVKUoiqIUiUociqIoSpGoxKEoiqIUiUociqIoSpGo\nxKEoiqIUiUociqIoSpGoxKEoiqIUiUociqIoSpGoxKEoSpGda1eHc+3qWDqMYqso7wNK971UyJnj\nilIWnXvyKQAq/+9bC0dyd1JK0OuRej0yNw/0ecg84w85uchbWcjsGyBzQUiyd20AJBgMICUgjdeQ\nEiTGYzL/2F8/YHwNCQaJ5K9jhr+CKDj292vG/0iD/Pc1/ypw2zF512N3XlOjuQECMudPLOr/qaL+\nr70tngc8fh9a7XVAFOsaplKJQ1HMTEqJ/soV9OcOgkGSuWGD8cM5Lw+Zm4fU50FeHjIv/7W8XOPx\n/GMy787j6PPPy8tF5uZCTrbxcc4t47m5Ocjc3L+P6/XGx3n6/Mf5iUFvyE8SEqk3GD+UDUV7b2ee\nGm6e/2mlxtjokvXBTAvHURK0aG30pXInlTgUpZiklOjT0shNSiY3OYnc5GRyk5PJSUoiN/k8ucnJ\nyOzsgvJJw4r+YSu0gAChASEkaCRC5P9oQGhk/nGJEOQfB43GeBwhEVoQVhJhB2g1CK0WYWUFWiuE\nzhphZYWw0oFOh7DSGV+ztgYra4S1Tf5zW7C2JWv1ckDg1L1//pdcgRAC480FaPL/BONrBcdACM2d\nr99eXtx5jkCA5q8W9duuk3/Pv44JIe48fvt5/zhmjNN47PKE9wHwfOW9Iv+d/OvLvfjnC6acZMI1\nTKxEXB73FkitaYWLSSUORTGBPj39X8kgN+nvJGG4ceOO8hpnZ3T+/thU8sCxsg5dbjw6UtDaGIyf\nX//4oBdWWrC2Q+hsEDY2CJ0d2NgibO2Nj3V2YGULOnvQ2eY//ut1u/zX7G577R7HrWxv+zB+MGm/\nbgDAeegHxbqOpRk++xQAu7b9LBxJ8Rk+/G+p3UvIYrarPdBNhXAHFgLBwFmgn5TyWiHlgoDZQCDG\nRsVYKeXZ+12/fv36Uq2OqxSFPiurIAn8lRBykpL/TgyZmXeU1zg4oAsIQOfvj87fH+sA/4LHOqt0\ntAlr4OgyuHLK+O22clOurDtKTqYtfrOW/vuDXau+wymWJYTYK6Wsb0pZS/1rfQNYJ6UcJ4R4I//5\n64WU+xb4UEq5RgjhSJFbYBXFyHDjhjEZJCfnNyndWWPQp6ffUV7Y2RmTQSV/7OvW/VeS0Li45DeP\nYOzUvHAIjv4Ma36Gq3HGZBHcDBoNg+pdwdEbD9nZWN6raim/e0UpWZZKHN2BlvmP5wEb+UfiEELU\nAKyklGsApJRZpRifUs4YsrPJPX9nE9LtSUJ/9eod5YWNTUEisK0difVtiUEXEIDWze3vxFAYKSHl\nIBz52ZgwrsbnJ4tHoMlIqNYVHP+xJ87TK8zwzhWl9FkqcfhIKVMApJQpQgjvQspEAGlCiCVACLAW\neENKWTrDBpQy4a8hrIFfzybv/Pk7awz5SSLnfDL61Mt3nqjToavkh7V/ALZt2txWY6iEdUAAWg8P\nRFHb+aWElAP5yWIZXDtj7LUOaQ5NX4RqXcDBs4TeuaKUXWZLHEKItYBvIYfGmHgJK+ARoA6QgLFP\n5D/A13e53xBgCEBQUFARo1XKIn1aGjkJCeivXuVEVPSd49y1WnR+fugCAnBs3jy/CSmgoMZg5eVV\n9MRQGCnh/H5jreLoMrh21pgsQltAs9H5ycKj+PdRlHLEbIlDStn2bseEEBeFEH75tQ0/4FIhxZKA\n/VLK+PxzfgYacZfEIaWcCcwEY+d4ceNXLMeQnc21777j8sxZGDIy0Lq54fb44/m1BmONwcrb2ziU\n1BykhOR9cHSpMVmkJYDGCkJbwiOvQLXOYO9unnsrSjlgqaaq5cBAYFz+n8sKKbMbcBNCeEkpU4HW\ngBoqVYFJvZ70ZctJ/eIL8i5cwLFFC7xefgnbiIhSuLmE5L1wZCkcXQ7pfyWLVtDidagaq5KFouSz\nVOIYB/wohBiEsRmqL4AQoj7wnJRysJRSL4R4BVgnjL2Ue4FZFopXMSMpJdc3b+bS+M+5dfIktpGR\nVPrkExxiGpr3xgYDJO8x1iqOLoP0RNDoIKw1tHwDqsWCnZt5Y1CUcsgi8zjMTc3jKD9uHjrMpfHj\nubFzJ7qgILxHj8KpY8d7j2gqDoMBknb/3WeRkQxaa2OyqNEDqnYCO1fz3FtRyrDyMI9DecjlJCSQ\nOmkSGStXoXVzw2fMGNwe7Wdc4qKkGQyQtOvv0VCZ5/OTRRto844xWdi6lPx9FaWCUolDKVV5V69y\nefqXXFu4EGFlhcew5/AYNAito2PJ3shggMQdxmRxbDlkpoDWBsLbQs33IKKDShaK8oBU4lBKheHm\nTa7Om8eVWbMx3LyJa+/eeI4cic6nsCk8D3oTPSTsyG+GWg5ZF4zJoko7YzNURAewdS65+ynKQ0ol\nDsWsZF4eaUuXcvmLKeSlpuLYpg3eL43GJiysZG5g0EPC9r9rFlkXjetA3Z4sbJxK5l6KogAqcShm\nIqUka8MGLn0+gZy4OOyiovCfNBH7evWKf3GDHs5tzU8Wv8D1S8aFAqu0g5o9oEoHsCnhpi9FUQqo\nxKGUuJsHDnDxs/Hc3LsX6+Bg/L+YjFO7dsUbKaXPMyaLo38li1Rjsohob6xZVGmvkoWilBKVOJQS\nc+vMGVInTiLz99/Renri++5YXHv3Ruh0D37Rma2NQ2YNeXDjsnE/iogOUKO7MVlYO5TcG1AUxSQq\ncSjFlnf5MqnTppH2408IGxs8R47E4+n/oHEo5od64m64cNC4R3SNHsZmqPB2YG1fMoErivJAVOJQ\nHpjh+nWuzJnLlblzkbdu4fZoPzyHD8fKswRWiD2+EhY9Ay6BMGAxeJRQZ7qiKMWmEodSZDI3l7RF\ni0idNh395cs4tW+P1+hR2ISElMwNds+Gla+CXzQ8/uO/97VQFMWiVOJQTCalJHPNGlInTCTn7Fns\n6tXDe8oX2NepU1I3gHXvwZaJENER+sxRfRiKUgapxKGY5MbevVz6bDw3DxzAOiyMgOnTcGzVquTW\nlMrLgeUj4c+FUO8/EPu52odbUcoo9Zup3NOtuDgufT6BrPXrsfLywveD93Ht2bNk98LIzoAfn4T4\njdD6LeOeF+Za5FBRlGJTiUMpVO7FS1yeOpW0xYvR2NnhNepF3J96Co19CY9oyjgP8/tC6nHo8SVE\nP16y11cUpcSpxKHcQZ+VxZXZs7n6zTykXo/bE0/gOew5rNzNsInRpWPwXR/ITjN2goe3Kfl7KIpS\n4lTiUACQOTlcW7CQy19+if7aNZxjO+E1ahTW5tq//ewWWPC4cV2pp1eCX5R57qMoSolTieMhJ6Uk\nc9UqLk2cRG5iIvYxMXi/8jJ2kZHmu+nhJbB0KLgFG+douN49OT06YzsAC4c2Nl88iqIUiUocD7Hr\nO3Zyafx4sg8fxqZKFQJnzsDhkUfMt/sewLap8PsYCGoM/b9X+3grSjmkEsdDKPvESS5N+Jzrm/7A\nytcXv48+wqV7N4RWa76bGgzGhLFjOlTvBr1mgc7WfPdTFMVsVOJ4iOSmpJD6xRTSf/4ZjaMj3q+8\njNuAAWhszfwBnpsNS4cYt22NGQYdPgTN/ZNUdq6epGs3SL+Zx5tLDxEd6EqdQFfCvBzRaNRwXUWx\nFJU4HgL6jAyuzJrF1W//BwYD7gMH4jF0CFZubua/+Y2rxk7whO3Q/kNoMtKk03aducobS/4kOS0b\nB2stvxw4z/c7EwBwtLGidoAL0YGuBT/ezqr2oiilRSWOCsyQk8O1+d9z5auv0Ken49y1K14vvoh1\ngH/pBJCWYBxue+2McfmQWr3ve0r6zVzGrTrOD7sSCHCzo6qPI6721vzwbCPiL2dxIDGdA4nXOJCY\nxsw/4skzSAAqudgSHWRMIlEBrkQGuGBvrf55K4o5qN+sCkhKScavv5I6cRK558/j0KQJ3q+8jG2N\nGqUXRMqfML8P5GXDk0shuNk9i0sp+e3wBcYuP8LlrFs8+0gIo9tF8PTc3QBoNIJwbyfCvZ3oUy8A\nMDZlHTmfzv6ENA4kpnEwKY2Vhy4AoNUIInyc8mskLkQHuhHu7YjWgk1caoSYUlGoxFHBSL2eC+9/\nQNrChQh7ewJnz8axWdPSDSJuPSx8Emxd4Zll4F39nsVT0m/yzrIjrDl6kZqVnPl6YAMiA1yAe3/I\n2uq01KvsTr3Kf4/Mupx1i4OJxkRyIDGNFX+e54ddxiYuB2stkQHGJBId6EqdIFd8VBOXohSZShwV\niCEnh/Ovvkbm6tV4PDsYr9GjERpN6QZx4AfjYoVe1eCJn8C50l2LGgyS73ae49PfTpBnMPB/naox\nqFkIVtoHj9nT0YY21X1oU92n4B5nrlznQIKxRnIgMY2vt8STqzc2cfk62xprJfnNXJH+LjjYqF8L\nRbkX9RtSQeizrpM0ciQ3duzA+7XX8Hjm6dINQErYPB7W/xdCWsCj/wNbl7sWP3kxkzcW/8m+hDSa\nhXvyUc9IgjxKfmc/jUYQ5uVImJcjve9o4sq4o2by2xFjE5dGcFsTlytRga5E+DhZtIlLUcoalTgq\ngLwrV0gcMpTs48fxG/cxrj16lG4A+jxY+QrsnQuR/aD7NLCyLrTorTw909af5stNcTjaWPF53yh6\n1fW/66TDp38zJsC5HeeWWLjGJi436lX+e1TZlaxb/JmUzv78RLLq8AUW7E4EwN5aS6S/i7FWEmCs\nnfi52JVYPIpS3qjEUc7lJCWTOHgwuSkpBEydglOrVqUcwHVYNAhOroJmo6HN2Lsuif7XENv41Ov0\niK7E211q4OFoU7rx3oWHow2tqnnTqpo3YOysP3vlhnEEV37n+5wtZwqauHycbQpqJNGBrtQOcMVR\nNXEpDwn1L70cyz55ksTBz2LIziZoztfY16tXugFkpcIPj8L5/RA7Hho+W2ixfw6xnfdMQ1pElO3t\nYIUQhHg6EOLpQM86xiauW3l6jp7PKGjeOpiYxuojF/PLQ4S3E1GBf3e+R/g4Fqu/RlHKKpU4yqkb\n+/aTOGwYGmtrKv/vf9hWjSjdAK7EwXe9ITMFHv0OqnUutNhvh1N4Z9mdQ2zL6/wKGystdYLcqBP0\ndxPXtes5HEhKK+gv+f3oRX7ckwSAne62Jq5AV27l6bFWiUSpACz2GyyEcAcWAsHAWaCflPJaIeU+\nBToDGmAN8KKUUpZepGVP1qZNJL04Cisfb4K+/hrrgIDSDSBpL3zfD6QBBv4CgQ3/VeRCejbvLDvM\n74UMsa1I3BysaVXVm1ZV/27iOnflBgeT0grml3yz9Sw5egNg7C9ZdiCZzpF+qjailFuW/Or3BrBO\nSjlOCPFG/vPXby8ghGgCNAVq57+0BWgBbCzFOMuU9OXLOf/mGGwiqhA0axZWHh6lG8CJVfDT0+Do\nDQOWgGf4HYcNBsn8nef4pASH2JYnQgiCPR0I9nSge7Rxhv6tPD3HUzJ54Yf9XMq8xYsLDvD57yd5\nrkUYvev5Y2NlxsUlFcUMLPnb3B2Yl/94HlDYUCAJ2ALWgA2gAy6WSnRl0NV58zj/2uvY16tH5W+/\nLf2ksWeOcd0p72oweO2/ksapi5n0nbGdt5cdITrQldWjmjO0RdhDkzTuxsZKS1SgK74utkT6OzPz\nyXq4OVjz5tJDPPLJBmb9Ec/1W3mWDlNRTGbJGoePlDIFQEqZIoTw/mcBKeV2IcQGIAUQwFQp5bFS\njtPipJSkTprMlRkzcGrXjkrjP0NjU4qjkaQ0zs/YPB6qtIc+c8HGseDwrTw90zbE8eXG0ziYMMT2\nYXXWejwA7Wsupl0NH7bFXWH6xtN8uPIYUzec5j9NgvlPk2DcHAofyqwoZYVZE4cQYi3gW8ihMSae\nHw5UB/5qxF8jhGgupfyjkLJDgCEAQeba7tQCpF7PhXffI+2nn3Dt2xffd8ead9+Mf9LnwvIX4OD3\nUPcp6DwRtH//s9l99ipvLP6TuBIYYiul5NKNS8SlxxGfFk9cehzHrx5HL/WM2zWORn6NqO9TH0dr\nx/tfrIwTQtA03JOm4Z4cSExj+obTTF53ilmb43m8YRCDHwnF10Uth6KUTcJS/cxCiBNAy/zahh+w\nUUpZ9R9lXgVspZQf5D9/B8iWUn56r2vXr19f7tmzx1yhlxrDrVucf+VVMtesweO5oXi9+GLpfovP\nzoAfn4L4DdDyTWjxWsEcjYxs4xDb73cm4O9qx4c9a9Gy6r8qjYWSUnLxxkVOp50mLi2O+PR4459p\n8WTmZhaUc7FxQW/QoxEacvQ5ZOuz0QotkZ6RNKrUiEZ+jajtWRudVmeWt1/SYuYaVwfe+fTiQo+f\nvJjJVxvjWHbwPFoh6F3Pn6HNwwj2dCjNMJWHlBBir5SyvkllLZg4PgOu3NY57i6lfO0fZR4FngU6\nYmyq+g2YJKX85V7XrgiJQ5+VRdKIkdzYuROfN/8P96eeKt0AMlLg+75w8Sh0+wLqDCg4dPsQ22ea\nhvBS+8KH2BqkgZTrKcSlxRX8/JUkbuTdKCjnbutOmGsYYS5hxj9dwwh1CcXd1p1nVj8DwIx2MziY\nepAdKTvYkbKDw5cPY5AG7KzsqO9Tn0Z+jWhUqRFVXKuU2Say+yWOvyRevcHMP+JZuCeRPL2BzrUr\nMbxlGNX9nEsjTJOolX4rnqIkDkv2cYwDfhRCDAISgL4AQoj6wHNSysHAIqA1cAhjR/lv90saFUHe\n5cskDBnCrZOnqPTZp7h07Vq6AaSeMM7RuHEVHv8RqrQF7hxiW8PPmdkD61M7wBW9QU9iRiJx6XGc\nTjtd0Mx0Jv0MN/NuFlzWy86LUNdQeoT3uCNBuNnef0Mpa601DXwb0MC3Ac/XeZ6MnAz2XNhTkEg+\n2/MZAB62HsT4xRgTiV8j/Bz9zPP/yIwC3e35oEctnm8TzpwtZ/luxzl+OXie1tW8GdEq7I7VgBXF\nEiyWOKSUV4A2hby+Bxic/1gPDC3l0CwqJymJhEGDyLt4icDp03Bs3rx0Azi3HX7oD1preHolVIo2\nDrHdlcCnq46Qo7lMv+ZaQv2TmR//Kx/sj+dM+hlu6W8VXMLH3ocw1zB6V+lNmGsY4a7hhLiE4GJT\ncvM4nK2daR3UmtZBrQG4cP0CO1N2sj1lOzvO72DlmZUABDsHE+MXQ2O/xtT3rV+iMZibt5Mtb3Sq\nxrAWYXy7/Sxzt52l95fbaRjizohW4TSv4llma1dKxVY+p/BWUNknTpI4eDCGnByC5s7Bvk6dB77W\nAy0OeORnWDKEXNdAErtPIi43lV1bv2DF8QOk5SViFXIZa5HHqlQgFSo5VCLUNZQY35iCGkSISwhO\n1k4PHPeD8nXwpXt4d7qHd0dKyem00wW1keVxy1l4YiEaoaGmR82C2ki0dzTW2tIbwVTjAZuaXOx1\nPN+mCoMeCWHh7kRm/hHPwDm7qOXvzPCW4XSo6atW71VKlUocZcSNvXtJHDYcja0twd/9D5sqVcx+\nzxx9DucyzhlHMR1aSFzCH8QF+nNOoydvw3AApBQI6U51rzCaBnW6o4nJXlfyy6CXBCEEVdyqUMWt\nCk/WeJJcQy6HUg8VJJI5h+cw69AsbLW21PWpW5BIqrpXRSPK7pwTe2srnm4awhMxlfl5fzJfbYpj\n+Px9hHo68FzLMHpE+2NtVXbjVyoOlTjKgMwNG0geNRqdnx9BX89G51+ye4Lf0t/ibPpZYwf1bUNd\nEzIS0Es9ABopCXByJcy/EdVEIJuPakm57EJs1dq826cOnmVkFdsHodPoqOtTl7o+dRkePZysnCz2\nXtxbkEgm7J0AgKuN6x39IwFOpbyUi4msrTT0axBI73oB/Hb4AtM3nua1RX8yac1Jnm0eSv8GQdhZ\nq9noivmoxGFhaT//TMqYt7CtXp3AmTOwci9+x+fV7KskZyVzM+8mXZZ2ITEzEYM0rpWkFVoCnQIJ\ncw2jXWBrwk6uJ+zMNipHDSSn1Yd8svoU8/OH2H79WK2CNZgqEkdrR1oEtqBFYAsAUm+kFiSRHSk7\nWH12NQABjgEFw35jfGNwtXW1ZNj/otUIOtf2IzbSl00nU5m+MY73fjnKlPWneaZpME82DsbFrnwM\nVVbKF4sNxzWn8jIc98qcuVz69FPsGzciYMpUtI7FG68vpWTlmZWM2zWOtFtp2GpteSTgkYLmpTCX\nMCo7Vza269+8BgsGwLkt0O59fnPux9hfjpCaaRxiO7pdxEO5haqUkjMZZ9hx3phEdl/YTVZuFgJB\nNfdqBYmkrnddbK2KNkHPHJtS/dOes1eZvjGO9ccv4WhjxYBGlRnULAQvp5KtMarhuBVPuZjHYU5l\nPXFIKUmdMIErs2bj1KEDlT77FI118TppL1y/wAc7PuCPpD+o7VUbvUGPnZVd4R9SaYkwvw9ciSO9\n4xe8diKC1UeMQ2zH9Y6kdkDZ+mZtSXmGPI5cOVKQSA6kHiDPkIe1xpo63nUKEkl19+poNfduHiqN\nxPGXo+cz+HJTHCv+PI9Oq6Ff/UCGNA8l0L1k+qVMnZOilB/lZR7HQ0nm5ZEydizpi5fg2v9RfN9+\nu1hLiBikgUUnFzFh7wQM0sDrDV7nsWqPMfj3wYWfcOEQzO+LzLnO2npf8tIKZ3L0qbyRv4qt7iFf\nkPCfrDRWRHlFEeUVxdCoodzIvcG+S/sKEsnkfZOZzGScrJ2I8Y0pmIgY5BRk0aGyNSo5M+WxOrzU\nLoIZm+JYsDuB73cl0D2qEsNahlHFp/RHvikVh0ocpciQnU3yy6+QtW4dnsOH4/n8yGJ9uJxNP8u7\n299l78W9NPJrxNjGY+/doRu/ERYMIFfnwOsO41iy2Z6m4S581DOSyh5qWQtT2OvsaebfjGb+zQC4\ncvMKuy7sYkfKDraf387ahLUA+Dn4FXSyx/jF4GFXyisZ5wvxdGBc79q82LYKszef4fudCSzZn0z7\nGj6MaBVOVKCqXSpFp5qqSok+M5OkYcO5sWcPPmPG4P7kgPufdBd5hjy+Pfot0w9Mx1przav1X6VH\neI97J6GDC5HLhnPFtjI900eTaePDmNjq9KkXoCaRlRApJYmZiQWd7DtTdpKRkwFAhFsE6bfScbFx\n4ccuP963Wctcrl3P4ZttZ/lm21nSb+bSNNyDES3DaRzmUaR/B6qpquJRfRxlLHHkpaaS8OwQbp0+\nTaVx43DpUvg2q6Y4fvU472x9h2NXj9EmqA1jYsbgZX+P/bulhC0TYd177NdG8tT1F2kVFc47XWuU\n6yG25YHeoOf41ePG2ewpO9iVsguJxMvOi44hHekc0pkaHjUskrizbuXx/c5zzN58hkuZt4gKdGVE\nyzDaVvdBY8JkQpU4Kh6VOMpQ4shJTCRh0GDyUlMJ+OILHB9p9kDXuaW/xYyDM5hzeA6uNq6MaTSG\ndpXb3fskg56cX17Gev9clumbMNF+FGN71amQQ2zLg6dWPUX6rXSCnYPZnLyZXEMuwc7BxIbE0imk\nE8EuwaUeU3aunsX7kpixKZ6EqzeI8HFkWMswutaudM8NuCpK4lCjw/6mOsfLiOzjx0l49lnIyaXy\nN3Oxi4p6oOvsv7Sfd7a+w9mMs3QP686rDV69/5pLOTe4+M0AfM6vY0ZeVy7FvMGK9tUeyiG2ZYVW\naHG3dWdy68mk30pnXcI6Vsav5MuDXzL94HRqetQkNiSWjiEd8bYvneRuq9PyRExlHq0fyIpDKUzf\nEMfohQeZsOYkQ5qH0bdeALY6NZlQuZP6FDGTG3v2GJcQcXAg6PtvsAkLK/I1rudeZ/K+ySw4vgA/\nBz9mtJ1BE/8m9z3v2qUkrn3dl+DsY0y3H0rTx99UnaBljIuNC72q9KJXlV5cvH6R387+xsozK/ls\nz2eM3zOehr4NiQ2NpW3ltjhbm385dSuthu7R/nStXYn1xy8xbeNp3v75MF+sO8WgZiE8EROEk62a\nTKgYqcRhBpnr15M8+iV0lSoZlxCpVKnI19iSvIX3t7/PhesXeLz647xQ5wWT1oaK278Rp2XP4Ccz\necvmNd575Q01xLaM83HwYWDNgQysOZAz6WdYdWYVK+JXMHbbWP6747884v8InUM70zygeZEnHRaV\nRiNoW8OHNtW92RF/lekbTzNu1XGmbzjNwCbBPN00BHe1te1DTyWOEpa2ZCkpb7+NbY0axiVE3O6/\n18Qd52en8dmez1get5wQlxC+7fQt0d7RJp27d8kkIg9+QKrwIKHnz3wU3fRB3oJiQSEuIQyPHs6w\nqGEcvXKUX+N/5bezv7E+cT0OOgfaBLWhc0hnGvo1xEpjvl9fIQSNwzxoHObBn0lpTN8Qx9QNp5m9\n+Qz9Gwaiz3VAq7tutvsrZZtKHCXoytdfc+mz8Tg0aULAlC/QOJg+N0JKye/nfuejnR+RcSuDobWH\nMqT2EJOW/c69dZODs4ZS//IyDtrWJWDwD/h7FbbVu1JeCCGo6VmTmp41eaX+K+y+uJuV8StZe24t\ny+OW427rTsfgjsSGxlLbs7ZZR2bVDnDlqyfrcfpSJl9ujOd/28+RZ+iDncspktNu4u9qZ7Z7K2WT\nGlVVAqSUXPpsPFfnzME5thN+48YVaQmRSzcu8eGOD1mfuJ4aHjV4v8n7VHWvev8Tgcvnz3B1bn8i\nco+z1fdJYgZNxEqn2qLLopJYcuSW/hZbkraw4swKNiVuIseQQ4BjAJ1COtEltAuhrqElFe5dJV27\nQbsZ07iZXhWdxor+DYIY3iozNqJqAAAgAElEQVQMP5fyl0AqyuiwkqBGVZUimZdHytvvkL50KW6P\nP4bPmDEmLyEipWTp6aWM3z2eHEMOL9d7mQE1BpjcBHFi12o8Vw7BX95kV8NJNO38dHHeilIO2Ght\naFO5DW0qtyEzJ7NgZNbXh79m1qFZVHOvVjC819fBPLXOADd7nH124OB+iA5er7JgdwIL9yTyeMMg\nhrcMw9vZvP0wiuWpxFEMhuxskke/RNaGDXiOHInniOEmNxkkZiby3rb32HlhJ/V96vNek/cIcg4y\n7cZSsuenT4g68ikXNN5c67uYhjVN+qKgVCBO1k70CO9Bj/AeXL55mdVnV7MyfiUT9k5gwt4J1POp\nR2xILO0rtzfLkvBa3XU+7BnJcy3CmLbhNP/bcY4fdiUwoFFlnmsRVuIr8iplh0ocD0ifkUHi8OHc\n3LsPn3fexv3xx007z6Bn/rH5TNk/BSuNFe80fofeVXqbvPPcrZtZHJ4xiPppv7HPvhGhQ+YT6OZZ\nnLeiVACedp48Uf0Jnqj+BIkZiaw8s5IVZ1bwwY4P+HjnxzT1b0psSCwtA1uW+M6Nge72jOtdm+Et\nw5my/hTfbDvL/J3nGNg4mCHNQ/FQKxRUOCYlDiFEU+CAlPK6EGIAUBeYLKU8Z9boyqjcS5dIfHYI\nt+Lj8Z/wOc6dOpl03qlrpxi7bSyHLh+iRUAL3mr0VpGaEy4lniRj3mPUyzvN1oBnafT0J2iLsbKu\nUjEFOgcyNMo4uOL41eOsPLOSlWdWsilpE3ZWdrQOak1sSCyNKzVGpym5/rAgD3s+6xvF8FbhTFl3\nilmb4/nfjnP8p0kwzz4SipsaxlthmFrj+BKIEkJEAa8BXwPfAi3MFVhZlXPunHEJkatXCfzqSxyb\n3n/Ia64+l1mHZjHr0CycdE582vxTOgZ3LNJImKNbl1NpzXB8ZB57m35F0/aPFedtKA8BIQTVPapT\n3aM6o+uNZu/Fvaw8s5Lfz/7OivgVuNq40iG4A7EhsUR7R5fYfushng5MeDSaEa3D+WLdKb7cFMe3\n28/xdNNgBjcLxcVeDd4wh9JcPsXUxJEnpZRCiO4YaxpfCyEGmjOwsij76FEShgyFvDwqz/sGu8jI\n+57zZ+qfjN02ltNpp+kc2pnXG7yOm63pczukwcDuH96n3slJJGoDEP3nUy/iwZYuUR5eGqGhgW8D\nGvg24M2Gb7L1/FZWxK9g2ellLDyxED8HPzqFdCI2JNbkEX33E+blyOT+dRjZKpxJ604xZf1pvtl6\nlkGPhPBMsxCc1Uz0csvUxJEphPg/YADQXAihBR6qv/Xru3aRNHwEGicngr6dh03ovYc93si9wdQD\nU/nu6Hd423szrc00mgc0L9I9b2ZlcGzGUzTM3MAeh+ZEPPctzs5Fm1CoKP+k0+poGdiSloEtuZ57\nnfUJ61l5ZiXzjsxjzuE5hLuG0zm0M51COuHv6F/s+1XxcWLa43V5vnUGk9acYtLaU8zZcoZnHwnl\nP02D1VIm5ZCpieNR4HFgkJTyghAiCPjMfGGVLZlr15L80svoAgONS4j43rtfYkfKDt7d9i7JWck8\nWvVRRtUdhaO1Y5HumRJ/hFvfPUaUPoGtISNp/OQHaNTSIeVaaWwZW1QOOge6hnWla1hXrmZf5fez\nv7PyzErjzob7JhPtFU1sqHFkVnE3o6rm68xXT9bjcHI6k9ae4vM1J/l66xmGNA9lYONgtQBnOWJy\njQNjE5VeCBEBVAN+MF9YZUfaokWkvDMW28haBH711T2XEMnIyeDzPZ+z5NQSKjtXZm6HudT3Lfow\n2UMbF1F54wvYITjUag5NW/YqzltQFJO427rTv1p/+lfrT3JWcsGaWR/t/IhPdn1Co0qN6BzSmdZB\nrYt1n1r+LsweWJ8/k9KYtPYUn/52gtmbzzC0eShPNq6MvbVKIGWdqX9DfwCPCCHcgHXAHoy1kCfM\nFZilSSm5Mns2qZ9PwKFZMwK+mIzG/u7DGNclrOPDHR9yNfsqg2oN4rmo54q8IJ006Nn17Vs0OPMl\n8doQbAd8T3Ro9eK+FUUpMn9HfwZHDmZw5GBOXjvJynjjyKw3t7yJrdaWXGGFlXQm15D7wCOzage4\nMuc/DdifcI2Ja0/x8arjzNocz3MtwhjQqLJazr0MMzVxCCnlDSHEIGCKlPJTIcQBcwZmSdJg4NKn\nn3H1m29w7tyZSh9/hLjLEiKXb17m450f8/u536nmXo2pbaZSw6NGke+ZlXGVuBkDiLm+lV3Obak5\ndC4OjuZfTltR7ifCLYKIehG8UPcFDqYeZEX8ChYe/wm9Jou2P7UlNiSW7uHdqeZe7YGuXyfIjW+f\nacjec1eZuOYU/11xjBl/xDO8ZRiPNQxSCaQMMjlxCCEaY6xhDMp/rUL+bcrcXFLeepv0ZctwGzAA\nnzf/D6H5d9+ClJLlccv5dPenZOdl82LdFxlYc+ADfftKPHkAueBxaupT2BbxCo0fG1PoPRXFkjRC\nQx3vOtTxrsPyY/vQy+vU9a7BghML+O7Yd0S4RdAtrBudQzvjaVf0San1Krvz3eAYdsZfYeLak7z3\ny1FmbIpnRKsw+jUIxMaqQn7klJiz1uPzH5l/3S1TE8co4P+ApVLKI0KIUGCD+cKyDMPNmySPGk3W\npk14vfgCHs89V+hci+SsZN7f/j7bzm+jjncd3m3yLqEuD7a43ME13xG25RVyhI5j7b+jSdMH349c\nUUqLQGCFIxNbTSQtO41VZ1fxS9wvjN8znol7J9KkUhO6hXWjVVArbLRFmzkeE+rBgiGN2RZ3mYlr\nTvL2siN8uTGOEa3D6VsvEGsr9aXK0kxKHFLKTcAmIYSTEMJRShkPvGDe0EqXPiuLxCFDubl/P77v\nvotb/0f/VcYgDfxw/Acm75uMQDAmZgz9qvZ7oIlThrw8dn/zKjFJczihi8D5qR+IDAovibeiKKXK\n1daVx6o9xmPVHiM+LZ7lccv5Jf4XXv3jVZx0TnQI6UC3sG5Ee0UXadJrkzBPGod6sPX0FSasOcGY\npYeZviGOF9qE06tugNqgzIJMXXIkEuNMcXfjU5EKPCWlPPIgNxVC9AXeBaoDDaWUha6BLoToCEzG\n2Cw2W0o57kHuZwqNjQ1WHh74T5yIc8cO/zoenxbP2G1jOZB6gKb+TRnbaCx+jn4PdK/0a6kkzHyM\nmJu72enamaihs7C1M33vDkUpq0JdQxlVbxTP13meXRd2sTxuOSviV7Do5CKCnIIKhv6aOj9ECEGz\nKp40Dfdg08lUJq49xeuLDzFtQxwvtKlCj+hKWKkEUupMbaqaAbwkpdwAIIRoCcwC7r8BduEOA73y\nr1uo/EmG04B2QBKwWwixXEp59AHveU9Cp8P/i8n/+kaUa8jlm8Pf8OXBL7HX2fNRs4/oEtrlgTfO\nOXt0J7qfnqKqIZXtNd+iUZ+XVX+GUuFoNVoaV2pM40qNuZ57nTXn1rA8bjnTDkxj2oFpNPBtQNfQ\nrrQPbo+D7v5fmoQQtKzqTYsILzacuMSENSd55aeDTNtwmhfahNMtyh+txnybWSl3MjVxOPyVNACk\nlBuFEA/8FVlKeQy434dvQ+B0frMYQogFQHfALImjsHiOXDnC2K1jOXHtBB2CO/B/Df+vWJOg9q+c\nTdWdb3Jd2HM69kcax7QtbsiKUuY56BwKln9Pzkrml7hf+CXuF97Z9g4f7/qYNkFt6BrWlRjfGLSa\ne3eACyFoXc2HVlW9WXP0IhPXnmL0woNMXX+aF9tG0DnSTyWQUmBq4ogXQrwN/C//+QDgjHlCKuAP\nJN72PAmIuVthIcQQYAhAUJCJ+1rcRXZeNtMPTufbI9/ibuvOpFaTaBPU5oGvp8/LZc/sF4i58D3H\nrGvg8Z8F1PCvXKwYFcWSavg92FBxf0d/not6jqG1h3Iw9SDL4pax+sxqfo3/FR97H7qEdqFbeLf7\nDjYRQtC+pi9tq/vw+9ELTFxzihd+2M+UdacY1TaCTrV80agEYjamJo5ngPeAJYDAOCHwntvNCSHW\nAoWtzTFGSrnMhHsW9rd+131upZQzgZlg3DrWhOsXaveF3by3/T3OZZyjd5XevFT/JZytH3w+RVrq\neZJnP0bMrQPs8OhFnSHTsbEpf1tsKkpJEkIQ7R1NtHc0bzR8gw2JG1h+ejnfHPmGrw9/TS2PWnQL\n70an4E733IRKoxF0rOVH+xq+rDycwqS1pxjx/T6q+Toxqm0EHWr6mHU/9oeVqaOqrlHEUVRSyuK2\nwyQBgbc9DwDOF/Oad3Uj9wZdlnYh9WYqAY4BzG4/mxi/u1ZwTHL6wGYcf36acJnGzqgPaNSrQg1E\nU5QSYaO1oWNwRzoGd+TyzcusiF/B8rjlfLTzIz7d/SktAlrQLawbj/g/gk5b+DwpjUbQpXYlOtXy\n49c/zzN57Sme+24vNSs5M6ptBG2re6sEUoLumTiEEL9w72/53Uo8or/tBqoIIUKAZKA/xoUWzcJK\nY0VWbhY+9j4s6b4EO6vi1Qr2/DyVyP3vck24cLb7EmLqFm1lXEV5GHnaeTKw5kAG1hzI8avHC0Zl\nrUtYh5uNG7GhsXQN60oN9xqFJgKtRtA92p/OkX4sP3ieyetO8ey3e6gd4MLothG0rOqlEkgJuF+N\nY/x9jj8QIURPYArgBawQQhyQUnYQQlTCOOw2VkqZJ4QYCazGOBx3zoMO/zWFtdaaLf23YK0t3i5l\nuTnZ7J81nIapizlsE4Xf4B+o6l38pakV5WFTzb0a1dyrMbreaLYlb2N53HJ+PPEj84/NJ9w1vGCW\nure997/OtdJq6FU3gG5RlViyP5kv1p3i6W92Ex3oyuh2ETSv4qkSSDHcM3HkT/wrcVLKpcDSQl4/\nD8Te9nwlsNIcMRSmuEnj8oUEUr/uT8PcI2z3eYz6g79Ap1PbZSpKceg0OloEtqBFYAvSb6Wz+uxq\nlsUtY8LeCUzaN4nGfo0LZqn/s6XASquhX/1AetbxZ/HeJKasP83AObuoV9mNl9pFICWo/FF0pk4A\nPMS/m6zSMa6S+18p5ZWSDqy8ObFnLe6/DiZYXmd3/fE07vqspUNSlArHxcaFflX70a9qP86mny2Y\npf765tdx1DnSPrg93cK6Ude77h01Cp1WQ/+GQfSqG8CPexKZtuE0T8zeic6uE46e+y34jsonU0dV\nrQL0wPf5z/tjHPWUDnwDdC3xyMoLKdm1+HOiD31EqsaT5N4LaBDZyNJRKUqFF+wSzAt1X2BknZHs\nvrCb5XHLWXVmFUtOLcHf0Z9uYd3oGtaVQKe/x9hYW2kY0KgyfeoFsHB3Iu+tuMG1xFg+XnWM1zpU\nU3NATGRq4mgqpWx62/NDQoitUsqmQogB5gisPLiVfZ0/ZzxLw2srOGhXn+Bnv8ffw8fSYSnKQ0Uj\nNMT4xRDjF8OYmDGsTVjL8tPL+ergV3x58EvqetelW1g32ge3x8naCQBbnZaBTYKZduw1MlMbMmMT\nnLyQyeTH6qi90E1g6loXjkKIgrGpQoiGwF97oeaVeFTlwMWk05wb35IG11awzf9par2yGheVNBTF\noux19nQL68bsDrNZ3Xs1L9R5gavZV3l3+7u0+rEVr/3xGluSt6A36AEQGj3OPtv5sGctNp+6TI9p\nW4lPzbLwuyj7TK1xDAbmCCEcMTZRZQCD8pcd+dhcwZVVR7etxPf3ofjLHPY1nkKTjk9ZOiRFUf7B\nz9GPZ2s/y+DIwRy6fKigKWvVmVV42XnRJbQLBm6hwYYnYioT7uXIsPn76D5tK1Mfr0uLCC9Lv4Uy\ny6Qah5Ryt5QyEogGoqWUtfNfuy6l/NG8IZYd0mBg5w//JWL1E2QJJ1IfW0VdlTQUpUwTQlDbqzZv\nNXqLDf02MKHlBGp61OTbo9+SrTlHtjjHjyd+pGaADctGNMXf1Y6n5+5i9uZ4pHzgRSgqNJMShxDC\nRQgxAeN+42uFEJ8LIVzMG1rZcvN6Jvsm9SPmxGcccmiE66gtBFera+mwFEUpAmutNe0qt2NKmyms\n67sOncELieSDHR/Q+qfWzDo2jvf7OdG+hg//XXGMl386SHau3tJhlzmm9nHMATKBfvk/GcBccwVV\n1pw/c5zzE5pTJ30t24KeI+rlX3F2cbd0WIqiFIOHnQc63LCVlfk+9ns6hXTit7O/MWjNU6S6fEz7\nRqdZcuA0/Wfu4GJGtqXDLVNM7eMIk1L2vu35e0KIA+YIqKw59MdSAtePxBEDh1rMoEnrf+8MqChK\n+SUQRHpFEukVyav1X2XlmZUsOrmI7emzca9mw6n0WnSeeYTZ/foSHeRm6XDLBFMTx00hRDMp5RYA\nIURT4Kb5wrI8aTCw87uxNIibQoI2CKvHvycqvJalw1KUMmFux4rZ4OBo7VgwwfDIlSMsPrmYX+J+\nJVu/lyd++5HOIT0Y0/xJXGweqpb6fzE1cQwD5uX3awjgKvAfcwVladcz0zgx4ykaZW1ir1NLqg2d\nh4PT3Zd2VhSl4qnpUZOajWvySv1XWHziV6bs/h+rzs9g9YI5dAxpT9+IPtTzqfdQrnll6rLqB4Ao\nIYRz/vMMs0ZlQYmnD6H//jGi9ElsDx9FoyfGqq1dFeUhZq+z58la/ehfvQ8v/byS3xOW8Ztcz8oz\nKwh2DqZPRB+6hXXDzfbhaca637LqL93ldQCklBPMEJPFHFi3gNDNo9Gj5WibeTRu3t3SISmKUkbo\ntBqm9O7C9ztr887yfXj7HcfG9SDj94xn8r7JtA1qS5+IPjTwbVDhayH3q3E4lUoUZUDa5QuE/zGK\nC1aVsH/yByKDq1o6JEVRyqDHY4II83Jg2HwHTlyuw/91dyUhd4NxguHZVQQ5BdE7ojfdw7rjYedh\n6XDN4n7Lqr9XWoFYmqunL8e7zie4Rgy29o73P0FRlIdWTKgHy0c25dlv9/LWj5f5v079WNfnRdYm\nrGXRyUVM3DuRKfum0CqoFX0i+tDIrxEaUXGavE3tHC8ghNgnpayQM9+q1W9j6RAURSknAtzsWTys\nMa/8dJAPVx7jWEoGH/Uy7lAYnxbP4lOLWR63nDXn1uDv6E/vKr3pEd4DL/vyv5TJg6TAit14pyiK\nYiJ7ayumPV6Xl9pFsGR/Mo/mTxYMdQ3l1QavsrbvWj555BP8Hf35Yv8XtFvUjlEbRrE5aXPBQovl\nUZFrHMCKEo9CURSlnBJC8EKbKkT4OPHSjwfoOmULM5+qT3SgKzZaG2JDY4kNjeVs+lmWnFrCsrhl\nrEtYh5+DH72q9KJneE98HMrXytpFrnFIKd8yRyCKoijlWcdaviwZ3gQbnYZ+M7azZF/SHceDXYJ5\nqf5LrO2zlvEtxlPZuTLTDkyj/eL2PL/ueTYlbiLPUD52qTB169hM7r517MtSyviSDkxRFKW8qebr\nzLIRzRgxfx8v/XiQ4xcyeb3jnTsL6rQ6OgR3oENwBxIzEllyeglLTy1lY9JGvO296VWlF73Ce+Hn\n6GfBd3JvptY4JgCvAv5AAPAKMAtYgHEBREVRFAVwd7Dm20ENeapxZWb+Ec8z3+wm/WZuoWUDnQN5\nse6LrOm7hkktJ1HFrQozDs6gw+IODF87nHUJ68g1FH6uJZnax9FRShlz2/OZQogdUsr3hRBvmiMw\nRVGU8kqn1fB+91pU83XmnWWH6TltK7MG1ifMq/Ch/jqNjjaV29CmchuSs5JZemopS08tZdSGUXjZ\nedEjvAe9qvQiwCmglN9J4UytcRiEEP2EEJr8n363HVM7nSiKohTi8Zggvn+2Eek3c+kxbSsbTly6\n7zn+jv6MrDOS1X1W80WrL6jhUYOvD39N7JJYhq4ZyppzayxeCzE1cTwBPAlcAi7mPx4ghLADRpop\nNkVRlHKvYYg7y0Y2JdDNnkHf7GbmH3Em7SxopbGiVVArpraZyureqxkWNYy4tDhe2vgSbX9qy8S9\nE0nISCiFd1BIbKYUyu/87nqXw1tKLhxFUZSKJ8DNnkXDGvPqT3/y0crjHEvJ5ONekdjqtCad7+vg\ny7DoYQypPYSt57ey6OQi5h2Zx5zDc4jxi6FPlT5IDIgHmppXdKaOqooAvgR8pJS1hBC1gW5Syv+a\nNTpFUZQKwt7aiqmP16Haeic+X3OS+MvXmflkPXycbU2+hlajpXlAc5oHNOfSjUv8fPpnFp9czKt/\nvApCixXO5OhzsNZam/GdmN5UNQv4PyAXQEr5J9DfXEEpiqJUREIInm9ThRlP1uP0xUy6TtnC/oRr\nD3Qtb3tvhtQewqreq/iq7VdoscPADXQaXQlH/W+mJg57KeWuf7xWPmaqKIqilDEdavqyZHhTbHQa\nHp25g8V7k+5/0l1ohIam/k2xkZWwkUGlsqS7qYnjshAijPwRVEKIPkCK2aJSFEWp4Kr6OrF8RDPq\nBbnx8k8H+XDFUfSG4g1SFaW0lKCpiWMEMAOoJoRIBkYBz5ktKkVRlIeAW/5kwYGNKzNr85l7ThYs\nS0xNHMnAXOBDjLPF1wADH/SmQoi+QogjQgiDEKL+XcoECiE2CCGO5Zd98UHvpyiKUlbptBre616L\nj3tFsi3uMj2nbSUuNcvSYd2TqYljGcbhuLnAeSALuF6M+x4GegF/3KNMHsZ1sKoDjYARQogaxbin\noihKmfVYw9smC041bbKgpZi65EiAlLJjSd1USnkMuGcnjpQyhfx+FCllphDiGMa1so6WVByKoihl\nSYNgd5Y/34xn5+3hmW9280bHagxpHlrm9jA3tcaxTQgRadZI7kEIEQzUAXbeo8wQIcQeIcSe1NTU\n0gpNUZRyrIafMzX8nC0dxh38Xe1YNKwxsbX8+HjVcV768SDZuWVr0ydTaxzNgP8IIc4AtzDuAiil\nlLXvdoIQYi3gW8ihMVLKZaYGKIRwBBYDo6SUGXcrJ6WcCcwEqF+/vlo/S1GUcuuvyYLVNzgx/veT\nxKdmMePJ+vi6mD5Z0JxMTRydinphKWXbop7zT0IIHcakMV9KuaS411MURSkvhBCMbG3cWXD0wgN0\nm7qFGU/Wo06Qm6VDM62pSkp5rrAfcwYmjI16XwPHpJQTzHkvRVEeTnM7zmVux7mWDuOe2udPFrTV\naYs9WbCklM6KWP8ghOgphEgCGgMrhBCr81+vJIRYmV+sKcZVeFsLIQ7k/8RaIl5FURRLqurrxLIR\nTalf2ThZ8L+/HiVPb7BYPKY2VZUoKeVSYGkhr58HYvMfb4FSmgapKIpSxrk5WDPvmYZ8uOIYs7ec\n4eSlLKb0r4OLvfnXpvoni9Q4FEVRlKLTaTW8260m43pFsj3uMj2mb+X0pdKfLKgSh6IoSjnTP3+y\nYGZ2Lj2nbWXD8dKdLKgSh6IoSjnUINidZSObEeRhzzPzdnP9SiQmbCxYIlTiUBRFKaf8Xe1Y9FwT\nYiP9yLrcgIyU5qUyWVAlDkVRlHLMzlrL1Mfq4Oi5B4PeDq2m7OzHoSiKopRRQggcPP7ENeB3dFrz\nf6yrxKEoilJBCFE6nRwqcSiKoihFYpEJgJaQm5tLUlIS2dnZlg6lTLK1tSUgIACdrvQnEymKUr48\nNIkjKSkJJycngoODy9za9pYmpeTKlSskJSUREhJi6XAURSnjHpqmquzsbDw8PFTSKIQQAg8PD1Ub\nUxTFJA9N4oB77zhYmEdnbOfRGdvNFE3ZohKqoiimeqgSh6U5OjoWPO7YsSOurq506dKl0LIjRowg\nOjqaGjVqYGdnR3R0NNHR0SxatKhI99y3bx+//fZbseJWFEW53UPTx1HWvPrqq9y4cYMZM2YUenza\ntGkAnD17li5dunDgwIEHus++ffs4fPgwHTuW2JbxiqI85FSNw0LatGmDk5PTA5176tQpOnToQL16\n9WjevDknT54EYMGCBdSqVYuoqChatWrFzZs3ef/995k/f/4D1VYURVEK81DWON775QhHz991+/IC\nR1OMZUzp56hRyZmxXWsWOzZTDBkyhNmzZxMWFsbWrVsZOXIkv//+O++99x4bN27Ex8eHtLQ07Ozs\neOeddzh8+DCTJk0qldgURan4HsrEUZ6lpaWxY8cOevfuXfBaXl4eAE2bNuWpp56ib9++9OrVy1Ih\nKopSwT2UicPUmsFfNY2FQxubM5wikVLi6elZaJ/HrFmz2LlzJ7/++itRUVH8+eefFohQUZSKTvVx\nlDNubm74+fmxdKlx512DwcDBgwcBiI+Pp1GjRnzwwQe4ubmRnJyMk5MTmZmZlgxZUZQKRiUOC3nk\nkUfo27cv69atIyAggNWrV5t87oIFC/jqq6+IioqiZs2a/PrrrwCMHj2ayMhIIiMjadu2LbVq1aJ1\n69YcPHiQOnXqqM5xRVFKxEPZVGUpWVl/7w28efNmk84JDg7m8OHDd7wWGhpaaKJZvnz5v17z8vJi\nz549RYxUURTl7lTiuIey1LehKIpSVqimKkVRFKVIVOJQFEVRikQlDkVRFKVIVOJQFEVRikR1jt/L\n3M7GP59eYdk4FEVR7iM455VSu5eqcZSi0l5WfenSpXz22WfFjltRFOV2qsZhISW1rHpeXh5WVoX/\nNfbs2bNkglUURbmNqnFYSHGWVW/WrBljxoyhefPmTJ06lWXLlhETE0OdOnVo3749ly5dAmD27NmM\nGjUKgAEDBvDiiy/SpEkTQkNDC5YsURRFKSqL1DiEEH2Bd4HqQEMp5V2nNgshtMAeIFlKWXi7TlGt\negMuHLp/uQv5iwT+1ddxL76R0Glc8eIqgoyMDP744w8Arl27Rrdu3RBC8NVXX/H555/zySef/Ouc\nS5cusXXrVg4dOkS/fv1UjURRlAdiqaaqw0AvoPB2mju9CBwDnM0aUTnTv3//gscJCQn069ePCxcu\ncOvWLSIiIgo9p0ePHgghqF27NsnJyaUVqqIoFYxFEoeU8hiAEOKe5YQQAUBn4EPgpRILwNSaQRke\nVeXg4FDweMSIEbz55pvExsaydu1axo0r/P3Z2NgUPJZSmj1GRVEqprLexzEJeA0w3K+gEGKIEGKP\nEGJPamqq+SMrQ9LT07Tru1cAAAwZSURBVPH390dKybx58ywdjqIoFZzZEocQYq0Q4nAhP91NPL8L\ncElKudeU8lLKmVLK+lLK+l5eXsWKvTQUZ1n1f3r33Xfp2bPn/7d3/0FWlfcdx98fFVlFHYgsG35o\nAnSJlB8rhRoYGIiGFAYzxMWKdgI0dByaGZ2mzmRqJR0VMYwMQ8dOZxqMRIVCRKshddBSaiNdZcqv\nMPxYu0QNMyrgBkpEJSVuCd/+cc9uQfbH/cm5u/fzmmHmnMtzzv0+u3C/8zzPud+HadOmUVNTU8Qo\nzcwuVLKpqoiYXuAtJgOzJc0CqoBrJK2LiHmFR5eOYpVVf+ONN847v/3228/bSrbV3Xff3Xa8bt26\nDmMxM8tF2X6PIyIeAB4AkPQV4LsXPWmU4dqGmVnaUlnjkFQv6TAwCXhZ0r8mrw+S9EoaMZmZWXbS\neqpqI3DBN9Ai4igwq53XtwJbSx6YmZl1qdyfqjIzszLjxGFmZjlx4ujEws0LWbh5YdphmJmVFSeO\ni6i1rPrevXuZNGkSo0aNYuzYsTz33HMXtC1GWXWAPXv2sHnz5qLEb2YGZfw4bk925ZVXsnbtWmpr\nazl69Cjjx49nxowZ9O3bt61NtmXVu7Jnzx4aGxuZOXNmUWI3M/OIIwUjRoygtrYWgEGDBjFgwABy\nKZPy9ttvM2PGDMaPH8/UqVN56623ANiwYQOjR4+mrq6Om2++mdOnT/PII4+wfv36vEYrZmbtqcgR\nx/Kdyzn464Ndtmttk806xw2fu4H7b7o/51h27txJS0sLw4cPz/qaRYsWsXr1aoYPH862bdu49957\n2bJlC0uWLGHr1q3U1NRw8uRJrrjiCh588EEaGxt5/PHHc47NzKw9FZk4ysUHH3zA/PnzWbNmDZdc\nkt3g7+TJk2zfvv28EiNnzpwBYPLkySxYsIA77riDOXPmlCRmM7OKTBzZjgxaRxpPz3y66DF8/PHH\n3HrrrTz66KNMnDgx6+sigv79+7e75vHkk0+yY8cONm3aRF1dHfv37y9myGZmgNc4UtHS0kJ9fX3b\n6CAX/fr1Y+DAgW1bv549e5Z9+/YBcOjQISZOnMjSpUvp168fR44c4eqrr+aTTz4peh/MrHI5caTg\n+eefp6GhgWeeeabtMdtcnprasGEDq1atoq6ujlGjRrFp0yYA7rvvPsaMGcOYMWOYPn06o0eP5pZb\nbmHfvn2MGzfOi+NmVhQVOVWVltZS5vPmzWPevOwK/bZXVn3YsGHt7t/x0ksvXfBadXU1u3d3uKW7\nmVnOnDg6UYq1DTOz7s5TVWZmlhMnDjMzy4kTh5mZ5cSJw8zMcuLE0Yl35y/g3fkL0g7DzKysOHFc\nRBe7rPrGjRtZsWJF0eI3MwM/jpuKYpZVP3PmDJdd1v6vsb6+vvjBm1nFc+JIwYgRI9qOzy2rfm7i\n6MyUKVOYNm0ar7/+OnPmzGHo0KEsW7aMlpYWqqurWbduHQMGDGD16tVtlXHnzZvHtddey65du2hu\nbmblypVOLGaWl4pMHM3LlvFpU9dl1X97MNMmm3WO3iNv4POLF+ccSz5l1SFTJLGhoQGADz/8kNmz\nZyOJVatWsXLlSpYvX37BNceOHWPbtm0cOHCAuXPnOnGYWV4qMnGUi3zKqre666672o7fe+895s6d\nS3NzM59++ul5I5pz3XbbbUhi7NixHDlypKDYzaxyVWTiyHZk0DrS+MI/ri16DPmWVW/Vp0+ftuN7\n7rmHxYsXM2vWLF599VUee+yxdq/p3bt323FE5B60mRl+qioVhZRVb89HH33E4MGDiQjWrFlThAjN\nzDrmxJGCQsuqf9bDDz9MfX0906ZNo6ampoiRmpldSD1xymLChAnx2VLiTU1NjBw5Mqf7lHKqqhzl\n8zMys/Jw5xP/CcBzfz4pr+sl/TwiJmTTtiLXOLJVKQnDzLq/fBNGPjxVZWZmOXHiMDOznKSSOCTd\nIelNSWcldTinJqmvpBckHZTUJKmgsVhPXM8pFv9szCxbaY04GoE5QEMX7f4O2BwRNwB1QFO+b1hV\nVcWJEyf8AdmOiODEiRNUVVWlHYqZdQOpLI5HRBOApA7bSLoGmAp8K7mmBWjJ9z2HDBnC4cOHOX78\neL636NGqqqoYMmRI2mGYWTdQzk9VDQOOA09LqgN+DnwnIn7TXmNJi4BFANdff/0Ff9+rVy+GDh1a\numjNzCpEyaaqJL0qqbGdP9/I8haXAX8A/CAixgG/Af66o8YR8cOImBARE6qrq4vQAzMza0/JRhwR\nMb3AWxwGDkfEjuT8BTpJHGZmdnGU7eO4EdEMvC/pS8lLXwX+K8WQzMyMlEqOSKoH/h6oBk4CeyNi\nhqRBwOqImJW0uxFYDVwOHAIWRsSHWdz/OPBunuH1B/47z2vLTU/pS0/pB7gv5ain9AMK68sXIiKr\nef4eWauqEJJ2Z1uvpdz1lL70lH6A+1KOeko/4OL1pWynqszMrDw5cZiZWU6cOC70w7QDKKKe0pee\n0g9wX8pRT+kHXKS+eI3DzMxy4hGHmZnlxImjHZKWStovaa+kLcljwt2OpBVJZeH9kjZK6pt2TPnK\ntqJyuZI0U9IvJL0jqVt/kVXSU5KOSWpMO5ZCSLpO0mtJ5e03JX0n7ZjyJalK0k5J+5K+LCnp+3mq\n6kKSromIj5PjvwB+PyK+nXJYOZP0R8DPIuKMpOUAEXF/ymHlRdJI4CzwBPDdiNjdxSVlQ9KlwFvA\n18hURNgF/ElEdMsvtEqaCpwC1kbE6LTjyZekgcDAiNgj6Woy9fBu646/F2UqxvaJiFOSegFvkKnt\nt70U7+cRRztak0aiD9Ats2tEbImIM8npdqDblr+NiKaI+EXaceTpJuCdiDiUVHneAGRbs63sREQD\n8Ou04yhURHwQEXuS40/IbNswON2o8hMZp5LTXsmfkn1uOXF0QNL3Jb0PfBN4MO14iuDPgH9JO4gK\nNRh4/5zzw3TTD6ieStIXgXHAjs5bli9Jl0raCxwD/u2cOn9FV7GJo6vqvRHxvYi4DlgP3JtutB3L\npgqxpO8BZ8j0pWwVoaJyuWpv45luOYrtiSRdBbwI/OVnZhu6lYj4XUTcSGZm4SZJJZtGLOf9OEoq\nh+q9PwZeBh4qYTh566ofkv4U+Drw1SjzBa0iVFQuV4eB6845HwIcTSkWO0eyHvAisD4ifpJ2PMUQ\nESclbQVmktlttegqdsTRGUm155zOBg6mFUshJM0E7gdmR8T/pB1PBdsF1EoaKuly4C7gpZRjqnjJ\ngvKPgKaI+Nu04ymEpOrWpyYlXQFMp4SfW36qqh2SXgS+ROYpnneBb0fEkXSjyp2kd4DewInkpe3d\n8ekw6LiicrpRZU/SLOBx4FLgqYj4fsoh5U3Ss8BXyFRi/RXwUET8KNWg8iBpCvA6cIDM/3WAxRHx\nSnpR5UfSWGANmX9flwDPR8QjJXs/Jw4zM8uFp6rMzCwnThxmZpYTJw4zM8uJE4eZmeXEicPMzHLi\nxGGWB0mnum7V6fUvSBqWHF8l6QlJv0wqmzZI+rKky5Pjiv2irpUnJw6zi0zSKODSiDiUvLSaTNHA\n2ogYBXwL6J8URPx34M5UAjXrgBOHWQGUsSKpqXVA0p3J65dI+odkBLFJ0iuS/ji57JvAPyfthgNf\nBv4mIs4CJFV0X07a/jRpb1Y2PAQ2K8wc4Eagjsw3qXdJagAmA18ExgADyJTsfiq5ZjLwbHI8isy3\n4H/Xwf0bgT8sSeRmefKIw6wwU4Bnk8qkvwL+g8wH/RTgnyLibEQ0A6+dc81A4Hg2N08SSkuy0ZBZ\nWXDiMCtMeyXTO3sd4DRQlRy/CdRJ6uz/Ym/gt3nEZlYSThxmhWkA7kw20akGpgI7yWzdeXuy1lFD\npihgqybg9wAi4pfAbmBJUq0VSbWte5BIuhY4HhH/e7E6ZNYVJw6zwmwE9gP7gJ8Bf5VMTb1IZh+O\nRjL7pO8APkqueZnzE8ndwOeBdyQdAJ7k//fruBnodtVarWdzdVyzEpF0VUScSkYNO4HJEdGc7Jfw\nWnLe0aJ46z1+AjzQjfdbtx7IT1WZlc6mZHOdy4GlyUiEiDgt6SEy+46/19HFyaZPP3XSsHLjEYeZ\nmeXEaxxmZpYTJw4zM8uJE4eZmeXEicPMzHLixGFmZjlx4jAzs5z8H3ktbsjwPj/zAAAAAElFTkSu\nQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x53410400>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "\n",
    "# plot CV误差曲线\n",
    "test_means = grid.cv_results_[ 'mean_test_score' ]\n",
    "test_stds = grid.cv_results_[ 'std_test_score' ]\n",
    "train_means = grid.cv_results_[ 'mean_train_score' ]\n",
    "train_stds = grid.cv_results_[ 'std_train_score' ]\n",
    "\n",
    "\n",
    "# plot results\n",
    "n_Cs = len(Cs)\n",
    "number_penaltys = len(penaltys)\n",
    "test_scores = np.array(test_means).reshape(n_Cs,number_penaltys)\n",
    "train_scores = np.array(train_means).reshape(n_Cs,number_penaltys)\n",
    "test_stds = np.array(test_stds).reshape(n_Cs,number_penaltys)\n",
    "train_stds = np.array(train_stds).reshape(n_Cs,number_penaltys)\n",
    "\n",
    "x_axis = np.log10(Cs)\n",
    "for i, value in enumerate(penaltys):\n",
    "    #pyplot.plot(log(Cs), test_scores[i], label= 'penalty:'   + str(value))\n",
    "    pyplot.errorbar(x_axis, test_scores[:,i], yerr=test_stds[:,i] ,label = penaltys[i] +' Test')\n",
    "    pyplot.errorbar(x_axis, train_scores[:,i], yerr=train_stds[:,i] ,label = penaltys[i] +' Train')\n",
    "    \n",
    "pyplot.legend()\n",
    "pyplot.xlabel( 'log(C)' )                                                                                                      \n",
    "pyplot.ylabel( 'neg-logloss' )\n",
    "pyplot.savefig('LogisticGridSearchCV_C.png' )\n",
    "\n",
    "pyplot.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 用logisticRegressionCV实现正则化的logisticregression"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## L1正则"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "LogisticRegressionCV(Cs=[0.01, 0.1, 1, 10], class_weight=None, cv=5,\n",
       "           dual=False, fit_intercept=True, intercept_scaling=1.0,\n",
       "           max_iter=100, multi_class='ovr', n_jobs=1, penalty='l1',\n",
       "           random_state=None, refit=True, scoring='neg_log_loss',\n",
       "           solver='liblinear', tol=0.0001, verbose=0)"
      ]
     },
     "execution_count": 107,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.linear_model import LogisticRegressionCV\n",
    "Cs = [0.01,0.1,1,10]\n",
    "lrcv_L1 = LogisticRegressionCV(Cs=Cs, cv = 5, scoring='neg_log_loss', penalty='l1', solver='liblinear', multi_class='ovr')\n",
    "lrcv_L1.fit(X_train,y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 108,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{0: array([[-0.33523496, -0.26780136, -0.3137749 , -0.35974421],\n",
       "        [-0.3359328 , -0.26599963, -0.30860965, -0.36223418],\n",
       "        [-0.3359328 , -0.25300989, -0.24592026, -0.25500594],\n",
       "        [-0.33038106, -0.25350585, -0.26616366, -0.28745534],\n",
       "        [-0.33038106, -0.23858412, -0.25214641, -0.2778433 ]]),\n",
       " 1: array([[-0.59946742, -0.52449083, -0.55699318, -0.643267  ],\n",
       "        [-0.59837959, -0.53674248, -0.63945079, -0.82693658],\n",
       "        [-0.59837959, -0.51227189, -0.53247576, -0.58029054],\n",
       "        [-0.59876487, -0.51960553, -0.55410486, -0.60616083],\n",
       "        [-0.59876487, -0.50823098, -0.57233727, -0.67387822]]),\n",
       " 2: array([[-0.66182912, -0.55356871, -0.6019909 , -0.69924185],\n",
       "        [-0.66349062, -0.54431352, -0.606462  , -0.74615987],\n",
       "        [-0.66381923, -0.49713166, -0.48752917, -0.5304685 ],\n",
       "        [-0.6616775 , -0.5307265 , -0.55276418, -0.6188563 ],\n",
       "        [-0.66562954, -0.53082031, -0.60919812, -0.73759668]])}"
      ]
     },
     "execution_count": 108,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lrcv_L1.scores_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYsAAAEKCAYAAADjDHn2AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4wLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvpW3flQAAIABJREFUeJzt3Xd4VNX2//H3Sqe3hJYEQoeETgAF\nxIogUiwIKF7L1atgQbn2RvOrXssFGyLWq4KCYqEKAirFgoROQguhhRp6KOnr90cGfxFDZoBMTjJZ\nr+eZh5kz58x8ToZkzT777H1EVTHGGGMK4ud0AGOMMcWfFQtjjDFuWbEwxhjjlhULY4wxblmxMMYY\n45YVC2OMMW5ZsTDGGOOWFQtjjDFuWbEwxhjjVoDTAQpLaGioRkVFOR3DGGNKlOXLlx9Q1TB36/lM\nsYiKiiIuLs7pGMYYU6KIyHZP1rPDUMYYY9yyYmGMMcYtKxbGGGPcsmJhjDHGLSsWxhhj3LJiYYwx\nxi0rFsYYY9yyYmGMMSXYzDW7mbZql9ffx4qFMcaUUJv2pfLYV2v47Lft5OSoV9/LioUxxpRAqWmZ\nDJ64nHLB/owb1BY/P/Hq+/nMdB/GGFNaqCpPfL2G7QdPMvGujtSoGOL197SWhTHGlDAfLtnK7LV7\nebx7Ey5uUK1I3tOKhTHGlCB/bD3ES99v4OroGtzTtX6Rva8VC2OMKSH2p6Zx/+crqFO1LK/1b4WI\nd/sp8rI+C2OMKQGysnN44POVpKZl8tldHagYElik72/FwhhjSoBX5m7kj62HGDugFU1rVizy97fD\nUMYYU8zNWbeH9xYlcetFdbi+TYQjGbxaLESkh4hsFJFEEXkyn+fvEJEUEVnlut19xvMVRWSXiLzt\nzZzGGFNcJaUc59Gv1tAqsjLP9Yp2LIfXDkOJiD8wDugGJAPLRGS6qiacseoUVX3gLC/zPLDQWxmN\nMaY4O5mRxZCJKwj0F94Z1JbgAH/HsnizZdEBSFTVJFXNACYDfT3dWETaATWAH7yUzxhjii1V5alv\n1rJpfypvDGxDeOUyjubxZrEIB3bmeZzsWnamG0VkjYhMFZFIABHxA/4LPObFfMYYU2xN/H0701bt\n5t9XNaZr4zCn43i1WOR3AvCZM13NAKJUtSUwH/jEtfw+YLaq7qQAInKPiMSJSFxKSsoFBzbGmOJg\nxY7DjJ6ZwOVNwrj/8oZOxwG8e+psMhCZ53EEsDvvCqp6MM/D94GXXfcvBi4RkfuA8kCQiBxX1SfP\n2P494D2A2NhY7065aIwxReDg8XTun7SCGhVDGDugtdcnCPSUN4vFMqCRiNQDdgEDgVvyriAitVR1\nj+thH2A9gKoOyrPOHUDsmYXCGGN8TXaO8tDkVRw8kcE3QzpRuWyQ05H+5LVioapZIvIAMBfwBz5S\n1XgRGQ3Eqep0YKiI9AGygEPAHd7KY4wxxd3YeZtYkniAl29sQfPwSk7H+QtR9Y2jN7GxsRoXF+d0\nDGOMOS8L1u/jrk/i6B8bwSv9WhXZ+4rIclWNdbeejeA2xhiH7Th4kmFTVhFTuyKj+zZ3Ok6+rFgY\nY4yD0jKzGTxxOQDjB7UjJNC5gXcFsYkEjTHGQcOnrSNhzzE+vD2WOtXKOh3nrKxlYYwxDpmybAdf\nxiXz4BUNubJZDafjFMiKhTHGOGBt8lGemxZPl4ahPHxVY6fjuGXFwhhjitiRkxkMmbSc0HJBvDGw\nNf7FZOBdQazPwhhjilBOjjJsyir2HUvjy3svplr5YKcjecRaFsYYU4Te/imRnzamMLxXNG3qVHE6\njsdKfbE4dCKDR79azca9qU5HMcb4uEWbUhg7fxPXta7NrRfVdTrOOSn1xUKA+ev3MWL6OnxlNLsx\npvjZdeQUD01eSePqFXjxhhaIFP9+irxKfbGoUi6Ix7o34fekQ8xcs8f9BsYYc47Ss7K5b+JyMrOV\n8be2pWxQyesuLvXFAmBg+zo0D6/IC7PWcyI9y+k4xhgf8/zMBFYnH+W1m1pSP6y803HOixULwN9P\nGNWnOXuPpfHWj4lOxzHG+JBvViQz8fcd3Nu1Pj2a13I6znmzYuHSrm4V+rWL4MMlSWxJOe50HGOM\nD1i/5xhPf7uWDvWq8lj3Jk7HuSBWLPJ4okdTQgL8GTk93jq7jTEX5FhaJkMmLqdiSCBv39KGAP+S\n/ee2ZKcvZGEVghnWrTGLNx/gh4R9TscxxpRQqsqjX65m5+FTjBvUluoVQpyOdMGsWJzhtovr0qRG\nBUbPSCAtM9vpOMaYEmjCoiR+SNjHU9c0pX1UVafjFAorFmcI8PdjVN8Ydh05xfiftzgdxxhTwvy2\n5SCvzNlAzxY1uatLPafjFBorFvm4qH41+rSqzfiFW9hx8KTTcYwxJcS+Y2k8+MUKokLL8Uq/ViVu\n4F1BrFicxdM9mxHgJzw/K8HpKMaYEiAzO4f7J63gZEY2E25tR/ngkjfwriBWLM6iZqUQhl7ZiHkJ\n+/hp436n4xhjirmXZm8gbvth/nNjSxrVqOB0nEJnxaIA/+xcj/qh5Rg9I4H0LOvsNsbkb+aa3Xz0\ny1bu6BRFn1a1nY7jFVYsChAU4MfIPjFsPXCCD5dsdTqOMaYYStyfyuNT19C2TmWe7tnM6TheY8XC\nja6Nw+geU4O3FiSy+8gpp+MYY4qR4+lZ3PvZcsoE+jNuUFuCAnz3T6rv7lkhevbaaHJUeWH2eqej\nGGOKCVXlya/XsPXACd66uQ21KpVxOpJXWbHwQGTVstx3WUNmrdnDr4kHnI5jjCkGPv5lGzPX7OHR\n7k3o1DDU6TheZ8XCQ/deWp/IqmUYMT2ezOwcp+MYYxwUt+0QL85ez1XNajC4awOn4xQJrxYLEekh\nIhtFJFFEnszn+TtEJEVEVrlud7uWtxaR30QkXkTWiMgAb+b0REigP8N7xbB5/3E++XWb03GMMQ5J\nSU3n/s9XEF6lDP/t3wo/P98ZeFcQrxULEfEHxgHXANHAzSISnc+qU1S1tev2gWvZSeA2VY0BegCv\ni0hlb2X11FXNqnNZkzBen7+Z/alpTscxxhSxrOwcHvxiBUdOZjJ+UDsqlQl0OlKR8WbLogOQqKpJ\nqpoBTAb6erKhqm5S1c2u+7uB/UCY15J6SEQY0TuGjKwc/vP9BqfjGGOK2Gs/bOL3pEO8cH0LomtX\ndDpOkfJmsQgHduZ5nOxadqYbXYeapopI5JlPikgHIAj426x+InKPiMSJSFxKSkph5S5QvdBy/Ktr\nPb5ZsYu4bYeK5D2NMc6bG7+Xdxdu4ZaOdejXLsLpOEXOm8UivwN5Z15RaAYQpaotgfnAJ395AZFa\nwGfAnar6t15lVX1PVWNVNTYsrOgaHvdf3pBalUIYPi2e7By7SJIxvm7rgRM8+uVqWkZUYniv/I6m\n+z5vFotkIG9LIQLYnXcFVT2oqumuh+8D7U4/JyIVgVnAs6r6uxdznrOyQQE8e200CXuO8fkfO5yO\nY4zxolMZ2QyZuBx/f+GdQW0JCfR3OpIjvFkslgGNRKSeiAQBA4HpeVdwtRxO6wOsdy0PAr4FPlXV\nr7yY8bz1bFGTTg2q8drcjRw6keF0HGOMF6gqz3y7lo37Unl9QGsiqpR1OpJjvFYsVDULeACYS24R\n+FJV40VktIj0ca021HV67GpgKHCHa3l/oCtwR57Talt7K+v5EBFG9YnhRHoWr861zm5jfNGkpTv4\nZuUuHrqyEZc1qe50HEeJqm8cc4+NjdW4uLgif9//m5nAh79s5bv7OtMq0vGze40xhWTVziP0f/c3\nLm5QjY/vaO+z4ylEZLmqxrpbz0ZwX6CHrmpEtXLBDJ8eT451dhvjEw6dyOD+SSsIqxDM6wNa+2yh\nOBdWLC5QhZBAnu7ZlNU7jzB1ebLTcYwxFyg7R3lo8kpSUtMZf2tbqpQLcjpSsWDFohBc3yac2LpV\neHnOBo6ezHQ6jjHmAryxYDOLNx9gZJ8YWkbYoeXTrFgUAhFhVN8YDp/MYOz8TU7HMcacp5827OfN\nBZvp1y6Cmzv8bYxwqWbFopDE1K7EoI51+fS3bazfc8zpOMaYc7Tz0EkenrKKZrUq8nzf5ohYP0Ve\nViwK0SNXN6Zy2SBGTIvHV84yM6Y0SMvMZsik5eSo8u6tbSkTVDoH3hXEikUhqlw2iMe7N+GPbYeY\nvnq3+w2MMcXCqBnxrNt1jDH9W1O3Wjmn4xRLViwKWf/YSFpFVOKFWes5np7ldBxjjBtfxu3kiz92\nct9lDegWXcPpOMWWFYtC5ucnjOrbnP2p6by5YLPTcYwxBVi36yjPfbeOTg2q8e9ujZ2OU6xZsfCC\n1pGVGRAbyUdLtpK4P9XpOMaYfBw9mcl9k1ZQpWwQb97chgB/+3NYEPvpeMnjPZpQNsifkdMTrLPb\nmGImJ0f595er2H3kFOMGtSW0fLDTkYo9KxZeUq18MI9c3YQliQeYs26v03GMMXmMX7iFBRv28+y1\nzWhXt4rTcUoEKxZeNKhjHZrWrMDzMxM4lZHtdBxjDLBk8wH++8NG+rSqze2dopyOU2JYsfCiAH8/\nRvdtzu6jabzzc6LTcYwp9XYfOcXQyStpEFael25oYQPvzoEVCy/rUK8q17WuzYSFSWw7cMLpOMaU\nWhlZOdw3aQXpmdmMv7Ud5YIDnI5UolixKAJP9WxGoL/w/MwEp6MYU2q9MCuBVTuP8OpNrWhYvbzT\ncUocKxZFoEbFEB6+qjELNuxnwfp9TscxptT5buUuPvltO3d3qUfPFrXcb2D+xopFEbmjcxQNq5dn\n9MwE0jKts9uYorJxbypPfbOW9lFVeOKapk7HKbGsWBSRQH8/RvaOYfvBk3ywOMnpOMaUCqlpmQyZ\nuJxywQGMu6UtgTbw7rx59JMTkc4iUs51/1YRGSMidb0bzfd0aRRKzxY1efunRJIPn3Q6jjE+TVV5\n7Ks1bD90knG3tKF6xRCnI5VonpbZ8cBJEWkFPA5sBz71Wiof9sy10QC8MGu9w0mM8W0fLN7KnPi9\nPNGjCR3rV3M6TonnabHI0tw5K/oCb6jqG0AF78XyXeGVy/DA5Q35ft1eFm9OcTqOMT5padJB/jNn\nAz1iavKvS+o7HccneFosUkXkKeBWYJaI+AOB3ovl2+6+pD51q5Vl5PR4MrJynI5jjE/ZfyyNB75Y\nSZ2qZXn1ppY28K6QeFosBgDpwF2quhcIB171WiofFxLoz4je0WxJOcH/ft3qdBxjfEZmdg73f76C\n42lZvHtrOyqE2HfawuJxy4Lcw0+LRaQx0Br4wnuxfN8VTWtwZdPqvDF/M/uOpTkdxxif8MqcDSzb\ndpiXbmhBk5p2pLwweVosFgHBIhIOLADuBP7nrVClxfDe0WRmKy/Nts5uYy7U7LV7eH/xVm67uC7X\ntQl3Oo7P8bRYiKqeBG4A3lLV64EYtxuJ9BCRjSKSKCJP5vP8HSKSIiKrXLe78zx3u4hsdt1u93SH\nSpK61cpx76X1+W7VbpYmHXQ6jjEl1paU4zz21WpaR1bmmWubOR3HJ3lcLETkYmAQMMu1zN/NBv7A\nOOAaIBq4WUSi81l1iqq2dt0+cG1bFRgBdAQ6ACNExCcnnb/vsoaEVy7DiOnxZGVbZ7cx5+pEehaD\nP1tOcKA/7wxqS3BAgX+azHnytFg8DDwFfKuq8SJSH/jJzTYdgERVTVLVDGAyuafeeqI7ME9VD6nq\nYWAe0MPDbUuUMkH+PNerGRv2pjJp6Q6n4xhToqgqT32zlsSU47w5sA21K5dxOpLP8qhYqOpCVe0D\nvCMi5V0FYKibzcKBnXkeJ7uWnelGEVkjIlNFJPIct/UJ3WNqckmjUP77w0YOHE93Oo4xJcanv21n\n+urdPNKtMV0ahTodx6d5Ot1HCxFZCawDEkRkuYi467PI7+TmMy9GPQOIUtWWwHzgk3PYFhG5R0Ti\nRCQuJaXkDnATEUb0juFkRjavzNngdBxjSoTl2w/zf7MSuLJpde67rKHTcXyep4ehJgD/VtW6qloH\neAR43802yUBknscRwO68K6jqQVU9/VX6faCdp9u6tn9PVWNVNTYsLMzDXSmeGlYvz11d6vFlXDIr\ndxx2Oo4xxdqB4+ncP2kFNSuFMKZ/a/z8bOCdt3laLMqp6p99FKr6M1DOzTbLgEYiUk9EgoCBwPS8\nK4hI3onl+wCnzyGdC1wtIlVcHdtXu5b5tAevbET1CsEMnxZPds7fGlLGGCA7Rxn6xUoOn8xg/KB2\nVCprA++KgqfFIklEnhORKNftWaDAoceqmgU8QO4f+fXAl67O8dEi0se12lARiReR1cBQ4A7XtoeA\n58ktOMuA0a5lPq18cADPXNuMtbuO8mXcTvcbGFMKjZm3kV+3HOT565rTPLyS03FKDcmdH9DNSrnf\n7kcBXcjtT1gEjHSdqVQsxMbGalxcnNMxLpiqMmDC72zen8pPj15G5bJBTkcyptiYl7CPf30ax8D2\nkfznxpZOx/EJIrJcVWPdrefp2VCHVXWoqrZV1Taq+lBxKhS+REQY1TeGo6cy+e8Pm5yOY0yxsf3g\nCf795Sqah1dkZB+3Y4JNIQso6EkRmUE+ZyGd5jqd1hSyZrUqctvFUXz62zYGtI+0prYp9dIysxk8\ncQV+Iowf1I6QQBt4V9QKLBbAa0WSwvzNsG6NmbF6NyOmxzN18MU2zbIptVSVZ79bx/o9x/j4jvZE\nVi3rdKRSqcBioaoLiyqI+atKZQJ5okdTHv96Dd+u3MUNbSOcjmSMIyYv28nU5ckMvaIhlzet7nSc\nUsvTQXlrXaOs894Wi8hYEbHrFXpJv3YRtI6szIuzN5Calul0HGOK3JrkI4yYFs8ljUJ56KrGTscp\n1Tw9dfZ7cicQHOS6zQAWA3uxqcq9xs9PGN03hoMn0nlj/man4xhTpA6fyGDIxBWElg/ijYFt8LeB\nd45y12dxWmdV7Zzn8VoR+UVVO4vIrd4IZnK1jKjMwPZ1+PjXbfRvH0njGnZBF+P7cnKUh6esIiU1\nnS8HX0zVcnYKudM8bVmUF5GOpx+ISAegvOthVqGnMn/xWPcmlA8OYMS0eDwZF2NMSffWj4ks3JTC\n8N7RtI6s7HQcg+fF4m7gAxHZKiLbgA+Au0WkHPCSt8KZXFXLBfFo9yb8lnSQWWv3OB3HGK/6eeN+\nXl+wiRvahDOoYx2n4xgXTwflLVPVFuRee7u1qrZ0LTuhql96N6IBuKVDHaJrVeSFWes5kW6NOeOb\nkg+f5OEpq2hSowIvXN/CThkvRjw9G6qSiIwh9/rb80XkvyJiI8WKkL+rs3vP0TTG/ZTodBxjCl16\nVjb3TVpBdrYy/tZ2lAmygXfFiaeHoT4CUoH+rtsx4GNvhTL5i42qyg1tw3l/cRJJKcedjmNMoRo1\nI4E1yUd5rX8r6oW6m9TaFDVPi0UDVR3hukJekqqOAup7M5jJ35PXNCU4wJ9RMxKss9v4jKnLk/l8\n6Q7uvbQ+3WNqOh3H5MPTYnFKRLqcfiAinYFT3olkClK9QggPX9WIhZtSmL9+v9NxjLlgCbuP8cy3\na7moflUeu7qJ03HMWXhaLIYA40Rkm4hsB94GBnsvlinI7Z2iaFS9PKNnxpOWme10HGPO29FTmQyZ\ntJxKZQJ56+a2BPh7+ifJFDVPz4ZapaqtgJZAC9c05au9G82cTaC/H6P6xrDz0CkmLExyOo4x5yUn\nR3nky9XsOnyKdwa1JaxCsNORTAHcTVH+77MsB0BVx3ghk/FApwah9GpZi3d+TuSGtuE2E6cpcSYs\nSmL++n0M7xVNbFRVp+MYN9y1LCq4uRkHPXNtM/xEeH5mgtNRjDknvyYe4NW5G7i2ZS3u7BzldBzj\nAXdTlI8qqiDm3NWqVIYHr2zIK3M28vPG/VzWxKZvNsXf3qNpPPjFSuqFluPlG1vawLsS4px7k0Rk\nhTeCmPNzV5d61Astx6gZCaRnWWe3Kd4ysnK4b9JyTmVmM+Ef7Sgf7OlcpsZp53PqgX0NKEaCA/wZ\n0TuarQdO8NGSbU7HMaZAL85ez4odR3j5xpY0rG5HskuS8ykWswo9hbkglzWpTrfoGrz142b2HLXh\nL6Z4mr56N//7dRt3do6id6vaTscx5+ici4WqPuuNIObCDO8VTVaO8uLsDU5HMeZvNu9L5cmv1xBb\ntwpP92zmdBxzHjydSDBVRI6dcdspIt+KiE37UQxEVi3LkEsbMGP1bn7bctDpOMb86Xh6FoMnLqds\nkD9v39KWQBt4VyJ5+qmNAR4DwoEI4FHgfWAyuZMMmmJgyGUNiKhShpHT48nMznE6jjGoKk9MXcPW\nAyd48+Y21KwU4nQkc548LRY9VHWCqqaq6jFVfQ/oqapTgCpezGfOQUigP8/1imbjvlQ++22703GM\n4aNftjFr7R4e79GUTg1CnY5jLoCnxSJHRPqLiJ/r1j/Pczb1aTFydXQNujYOY+y8TaSkpjsdx5Ri\ny7Yd4qXZ67k6ugb3drWj1SWdp8ViEPAPYD+wz3X/VhEpAzxwto1EpIeIbBSRRBF5soD1+omIikis\n63GgiHwiImtFZL2IPOXxHpVyIsLI3tGkZWXz8hzr7DbO2J+axv2TVhBRpQyv9W9lA+98gKcTCSap\nam9VDVXVMNf9RFU9papL8ttGRPyBccA1QDRws4hE57NeBWAosDTP4puAYNelXNsB94pI1LnsWGlW\nP6w8d19Sn6nLk1m+/bDTcUwpk5Wdw4Ofr+RYWibjb21HxZBApyOZQuDp2VCNRWSBiKxzPW4pIu5O\noe0AJLoKTQa5neF981nveeAVIC3PMgXKiUgAUAbIIPfqfMZDD1zekJoVQxg+bR3ZOXak0BSdV+du\nZOnWQ7x4fQua1arodBxTSDw9DPU+8BSQCaCqa4CBbrYJB3bmeZzsWvYnEWkDRKrqzDO2nQqcAPYA\nO4DXVPXQmW8gIveISJyIxKWkpHi4K6VDueAAnrm2GfG7j/HFHzucjmNKiTnr9jBhURKDOtbhhrYR\nTscxhcjTYlFWVf84Y1mWm23yO0j551dcEfEDxgKP5LNeByAbqA3UAx7JbzyHqr6nqrGqGhsWFuYm\nTunTq2UtLqpfldd+2MjhExlOxzE+LinlOI9+tYZWEZUY3vtvR5xNCedpsTggIg1w/bEXkX7kfusv\nSDIQmedxBLA7z+MKQHPgZxHZBlwETHd1ct8CzFHVTFXdD/wCxHqY1biICKP6NCc1LYtXf9jodBzj\nw05mZDFk4goC/YV3bm1HcIC/05FMIfO0WNwPTACaisgu4GHcX1Z1GdBIROqJSBC5h62mn35SVY+6\nOsyjVDUK+B3oo6px5B56ukJylSO3kNipPeehSc0K3H5xFF/8sYO1yUedjmN8kKryzLfr2LQ/lTcG\ntiG8chmnIxkv8LRY7AI+Bl4gt6N6HnB7QRuoaha5p9XOBdYDX6pqvIiMFpE+bt5vHFAeWEdu0fnY\n1U9izsPD3RpRrVwQw6evI8c6u00hm7h0B9+u3MWwqxrTtbEdDvZVnk4mPw04Aqzgr4eSCqSqs4HZ\nZywbfpZ1L8tz/zi5p8+aQlAxJJAnr2nGo1+t5usVydwUG+l+I2M8sHLHYUbPiOeyJmE8cHlDp+MY\nL/K0WESoag+vJjFedUObcD5fup2X52zg6piaVCpj576bC3PweDr3TVpBjYohvD6gNX5+NvDOl3l6\nGOpXEWnh1STGq/z8hNF9m3PwRAavz9/kdBxTwmXnKA9PWcXBExmMH9SOymWDnI5kvMzTYtEFWO6a\numONaxoO60MoYZqHV2JQxzp8+tt2Nuy1MY7m/L0+fxOLNx9gdJ8YWkRUcjqOKQKeFotrgEbA1UBv\noJfrX1PCPHp1EyqGBDB8Wjyq1tltzt2PG/bx1o+J3NQuggHtrf+rtPB0bqjt+d28Hc4Uvsplg3is\ne1P+2HqI6as9PlfBGAB2HjrJw5NXEV2rIs9f19wmCCxF7JJVpdCA9pG0CK/Ei7PXczzd3UB8Y3Id\nPJ7O4InLAXj31naEBNrAu9LEikUp5O8njOobw75j6bz142an45hiTlWZsXo33cYuYtO+VF4f2Jo6\n1co6HcsUMSsWpVTbOlW4qV0EHy3ZSuL+407HMcXU/tQ0Bk9czoNfrCSyShlmDb2EK5rWcDqWcYAV\ni1Ls8R5NCQn0Z9QM6+w2f6WqfL08mW5jFvHzxhSe7tmUr4d0onGNCk5HMw6xYlGKhVUI5t/dGrN4\n8wHmxu9zOo4pJvYcPcU//7eMR75aTaPq5fn+oUu4p2sDAvztz0Vp5ukIbuOj/nFRXSb/sZPnZyZw\naeMwygRZp2VppapMXraTF2etJytHGdE7mtsvjrKR2QawlkWpF+Dvx6i+Mew6corxC7c4Hcc4ZOeh\nk9z64VKe+mYtzcMrMffhrtzZuZ4VCvMna1kYLqpfjT6tavPuwi30axthZ7qUIjk5yme/584Z5ifC\ni9e34OYOkTZ+wvyNtSwMAE/3bEagnzB6ZrzTUUwR2XrgBAPf+50R0+OJjarK3GFduaVjHSsUJl9W\nLAwANSuFMPTKRsxfv58fN1hnty/LzlHeX5REj9cXsWHvMV7t15JP7mxvFy0yBbLDUOZPd3aux5S4\nnYyakUCnBqE2QtcHbd6XymNT17Bq5xGualaDF65vTo2KIU7HMiWAtSzMn4IC/BjZO4btB0/y4ZKt\nTscxhSgzO4dxPyVy7ZtL2H7wBG8MbM37t7WzQmE8Zi0L8xddG4fRI6Ymb/24mevahNuhCR+QsPsY\nj3+9mnW7jnFti1qM6htDaPlgp2OZEsZaFuZvnu3VDFV4cdZ6p6OYC5CRlcPYeZvo8/YS9h5NY/yg\ntowb1NYKhTkv1rIwfxNRpSz3X96QMfM2cUviATo3DHU6kjlHa5KP8PjUNWzYm8oNbcJ5rlc0VcrZ\n1ezM+bOWhcnXPV3rU6dqWUZMjyczO8fpOMZDaZnZ/Of7DVw37hcOn8zgw9tjGTOgtRUKc8GsWJh8\nhQT6M7xXNIn7j/PJr9ucjmM8sHz7IXq+uZh3F27hpnaR/DDsUq5sZjPEmsJhh6HMWV3ZrDqXNwnj\n9fmb6dOqNtXtzJli6VRGNq/9sJGPftlK7Upl+OyuDlzSKMzpWMbHWMvCnJWIMLx3DBlZOfzn+w1O\nxzH5+G3LQXq8sYgPl2zl1o4VjaNOAAASnElEQVR1mTusqxUK4xVWLEyB6oWW419d6/HNyl0s23bI\n6TjG5Xh6Fs99t46b3/8dgMn3XMTz1zWnfLAdLDDeYcXCuHX/5Q2pXSmE575bR5Z1djtu0aYUuo9d\nxMSl27mrSz3mPNSVi+pXczqW8XFeLRYi0kNENopIoog8WcB6/URERSQ2z7KWIvKbiMSLyFoRsQPm\nDikbFMCzvaLZsDeVz//Y4XScUuvoqUyemLqG2z76g5BAP6YO7sRzvaLtGiSmSHitzSoi/sA4oBuQ\nDCwTkemqmnDGehWAocDSPMsCgInAP1R1tYhUAzK9ldW4d03zmnRuWI3X5m7k2ha1qGYDu4rUgvX7\neObbdaQcT2fIZQ146MpGNneXKVLebFl0ABJVNUlVM4DJQN981nseeAVIy7PsamCNqq4GUNWDqprt\nxazGDRFhZO8YTmZk8+rcjU7HKTUOn8hg2JRV3PVJHJXLBvLtfZ14wnXtdGOKkjeLRTiwM8/jZNey\nP4lIGyBSVWeesW1jQEVkroisEJHHvZjTeKhRjQrc2TmKKXE7WbXziNNxfN6cdXvoNnYRM1bv5qEr\nGzH9gS60jKjsdCxTSnmzWOR3BRX980kRP2As8Eg+6wUAXYBBrn+vF5Er//YGIveISJyIxKWkpBRO\nalOgoVc2IrR8MCOmrSMnR91vYM7ZgePp3D9pBYMnrqBmpWCmP9CFYd0aExRg56MY53jzf18yEJnn\ncQSwO8/jCkBz4GcR2QZcBEx3dXInAwtV9YCqngRmA23PfANVfU9VY1U1NizMzi0vChVCAnm6Z1NW\nJx/lq+U73W9gPKaqTFu1i25jFjIvYR+PdW/Cd/d1Jrp2RaejGePVYrEMaCQi9UQkCBgITD/9pKoe\nVdVQVY1S1Sjgd6CPqsYBc4GWIlLW1dl9KZDw97cwTriudTjto6rw8pyNHD1p5x0Uhn3H0vjXp8t5\naPIqokLLMfuhLtx/eUMC/K01YYoHr/1PVNUs4AFy//CvB75U1XgRGS0ifdxsexgYQ27BWQWsUNVZ\n3spqzo2IMLJPDEdOZjBmnnV2XwhV5au4nXQbs5DFm1N49tpmTB3ciYbVKzgdzZi/EFXfOO4cGxur\ncXFxTscoVYZPW8fE37cz88FL7FDJedh15BRPfbOWRZtS6FCvKi/f2JJ6oeWcjmVKGRFZrqqx7taz\nNq45b490a0LlskGMmL4OX/nSURRycpRJS7fTfewi4rYdYnTfGCb/6yIrFKZYs2JhzlulsoE80aMJ\ny7Yd5rtVu5yOUyLsOHiSQR8s5Zlv19E6sjJzH+7KbRdH4eeX38mDxhQfNuuYuSA3tYvk86U7eHH2\nBq5qVoMKIYFORyqWcnKUT37bxitzNhLgJ/znhhYMaB+JiBUJUzJYy8JcED8/YXTf5hw4ns6bCzY7\nHadYSko5Tv8JvzFqRgIX1a/K3GFdGdihjhUKU6JYy8JcsFaRlRkQG8nHv2yjf2wkjWrYmTwAWdk5\nfLhkK2PmbSIk0J8x/VtxfZtwKxKmRLKWhSkUj3VvQtkgf0bOiLfObmDTvlRuHP8rL32/gUsbhzFv\nWFduaBthhcKUWFYsTKGoVj6YR7s34ZfEg3y/bq/TcRyTmZ3DWws2c+2bi9l5+BRv3dyGCf9oZ5ek\nNSWeHYYyheaWDnX44o+d/N/MBC5rEkbZoNL13yt+91Ee+2oNCXuO0btVbUb2jrap3I3PsJaFKTQB\n/n6M7hvD7qNpvPPTFqfjFJn0rGz++8NG+r79CynH05nwj3a8dXMbKxTGp5Sur37G69pHVeX6NuG8\ntyiJfu0iiPLxgWardh7h8amr2bTvODe2jeC5Xs2oXDbI6VjGFDprWZhC99Q1TQn0F0bP9N25H9My\ns3lp9npueOcXUtOy+PiO9vy3fysrFMZnWbEwha56xRAevqoxP27Yz/yEfU7HKXRx2w7R843FTFiU\nxID2dZg7rCuXN63udCxjvMqKhfGKOzpH0bB6eUbNjCct0zeuiHsyI4uR0+O5acJvZGTnMOnujrx0\nQwsq2qh1UwpYsTBeEejvx6g+Mew8dIr3FiU5HeeC/brlAN1fX8T/ft3G7RdHMffhrnRuGOp0LGOK\njHVwG6/p3DCUa1vUYtxPiVzfJpzIqmWdjnTOUtMy+c/3G5i0dAdR1cry5b0X06FeVadjGVPkrGVh\nvOrpa5vhJ8ILs9Y7HeWc/bxxP93HLuKLP3ZwT9f6fP9QVysUptSyYmG8KrxyGR64oiFz4veyaFOK\n03E8cvRkJo9+tZo7Pl5G2eAAvh7Siad7NqNMkL/T0YxxjBUL43V3X1KPqGplGTkjnoysHKfjFGhe\nwj66jV3Ityt3cf/lDZg1tAtt6lRxOpYxjrNiYbwuOMCfEb1jSEo5wce/bHU6Tr4Oncjgockr+den\ncVQtF8S0+zvzWPemBAdYa8IYsA5uU0Qub1qdq5pV580Fm+nbOpyalYrPxHqz1uxh+LR1HEvLZNhV\njRlyWQOCAux7lDF52W+EKTLP9YomM0d56fvi0dmdkprOkInLuf/zFYRXKcOMB7vw0FWNrFAYkw/7\nrTBFpm61cgzuWp9pq3bze9JBx3KoKt+uTKbb2IUs2LCfJ3o05ZshnWhas6JjmYwp7qxYmCI15LKG\nhFcuw4hp8WRlF31n996jadz9SRzDpqymfmg5Zg+9hCGXNSDA334VjCmI/YaYIlUmyJ/nejVj475U\nPvt9e5G9r6oyZdkOuo1ZyC9bDvBcr2i+GtyJhtXLF1kGY0oy6+A2Ra57TE0uaRTKmB820atlbcIq\nePe6D8mHT/LUN2tZvPkAHetV5ZV+LalbzbenTjemsFnLwhQ5EWFknxjSsrJ5Zc4Gr71PTo7y2e/b\n6T52ESu2H+b565rzxb8uskJhzHmwloVxRIOw8vyzSz0mLEzi5o51aFvIA9+2HTjBE1+vYenWQ1zS\nKJSXbmhBRJWSNzeVMcWFV1sWItJDRDaKSKKIPFnAev1EREUk9ozldUTkuIg86s2cxhkPXtGIGhWD\nGTEtnuwcLZTXzM5RPlicRI83FpGw5xiv3NiST//ZwQqFMRfIa8VCRPyBccA1QDRws4hE57NeBWAo\nsDSflxkLfO+tjMZZ5YMDeLpnM9buOsqUZTsv+PUS9x/npnd/5f9mradzg1DmDbuU/u0jEZFCSGtM\n6ebNlkUHIFFVk1Q1A5gM9M1nveeBV4C0vAtF5DogCYj3YkbjsD6tatOhXlVembuBwycyzus1srJz\nGP/zFnq+uZikAyd4fUBrPrg9tliNEjempPNmsQgH8n5dTHYt+5OItAEiVXXmGcvLAU8Aowp6AxG5\nR0TiRCQuJaVkzGhq/kpEGNUnhtS0LP47b+M5b79h7zGuf+dXXp6zgSuaVOeHYV25rk24tSaMKWTe\nLBb5/bb+eWBaRPzIPcz0SD7rjQLGqurxgt5AVd9T1VhVjQ0LC7ugsMY5zWpV5B8X1WXS0h2s23XU\no20ysnJ4Y/5mer+1hN1HTvHOoLa8+492VK9grQljvMGbZ0MlA5F5HkcAu/M8rgA0B352fQusCUwX\nkT5AR6CfiLwCVAZyRCRNVd/2Yl7joGHdGjNj9W6GT1vH1MGd8PM7e8tg3a6jPPrVajbsTaVv69qM\n6B1D1XJBRZjWmNLHm8ViGdBIROoBu4CBwC2nn1TVo8CfFzEWkZ+BR1U1Drgkz/KRwHErFL6tUplA\nnrimKY9PXcM3K3fRr13E39ZJz8rmzQWbeXdhEtXKBfH+bbF0i67hQFpjSh+vHYZS1SzgAWAusB74\nUlXjRWS0q/VgzF/0axtB68jK/Of79RxLy/zLcyt3HObaN5cw7qct3NAmnHnDLrVCYUwREtXCOb/d\nabGxsRoXF+d0DHOB1iQfoe+4X7izUz2G947mVEY2Y+Zt5MMlW6lZMYSXbmzJpY2tf8qYwiIiy1U1\n1t16NoLbFCstIypzc4c6fPLbNprULM/4n7ew7eBJBnWsw5PXNKVCSKDTEY0plWxuKFPsPHZ1EyqE\nBPDE12vJVuXzuzvywvUtrFAY4yBrWZhip0q5IMYOaM2qHUe499L6lA2y/6bGOM1+C02xdHmT6lze\npLrTMYwxLnYYyhhjjFtWLIwxxrhlxcIYY4xbViyMMca4ZcXCGGOMW1YsjDHGuGXFwhhjjFtWLIwx\nxrjlMxMJikgKsP0CXiIUOFBIcZzkK/sBti/Fla/si6/sB1zYvtRVVbezc/pMsbhQIhLnycyLxZ2v\n7AfYvhRXvrIvvrIfUDT7YoehjDHGuGXFwhhjjFtWLP6/95wOUEh8ZT/A9qW48pV98ZX9gCLYF+uz\nMMYY45a1LIwxxrhVaouFiLwqIhtEZI2IfCsilc+yXg8R2SgiiSLyZFHndEdEbhKReBHJEZGzng0h\nIttEZK2IrBKRYnmx8nPYl2L9mQCISFURmScim13/VjnLetmuz2SViEwv6pxn4+5nLCLBIjLF9fxS\nEYkq+pSe8WBf7hCRlDyfw91O5HRHRD4Skf0isu4sz4uIvOnazzUi0rZQA6hqqbwBVwMBrvsvAy/n\ns44/sAWoDwQBq4Fop7OfkbEZ0AT4GYgtYL1tQKjTeS90X0rCZ+LK+QrwpOv+k/n9/3I9d9zprOfz\nMwbuA9513R8ITHE69wXsyx3A205n9WBfugJtgXVneb4n8D0gwEXA0sJ8/1LbslDVH1Q1y/XwdyAi\nn9U6AImqmqSqGcBkoG9RZfSEqq5X1Y1O5ygMHu5Lsf9MXPoCn7jufwJc52CWc+XJzzjv/k0FrhQR\nKcKMniop/1/cUtVFwKECVukLfKq5fgcqi0itwnr/UlsszvBPcivymcKBnXkeJ7uWlUQK/CAiy0Xk\nHqfDXICS8pnUUNU9AK5/z3aN2BARiROR30WkuBQUT37Gf67j+tJ1FKhWJOnOjaf/X250HbqZKiKR\nRROt0Hn1d8Onr8EtIvOBmvk89YyqTnOt8wyQBUzK7yXyWVbkp495sh8e6Kyqu0WkOjBPRDa4vqkU\nqULYl2LxmUDB+3IOL1PH9bnUB34UkbWquqVwEp43T37GxeZzcMOTnDOAL1Q1XUQGk9tiusLryQqf\nVz8Tny4WqnpVQc+LyO1AL+BKdR30O0MykPdbRgSwu/ASesbdfnj4Grtd/+4XkW/JbZ4XebEohH0p\nFp8JFLwvIrJPRGqp6h7XoYD9Z3mN059Lkoj8DLQh9xi7kzz5GZ9eJ1lEAoBKFHyIxClu90VVD+Z5\n+D65fZglkVd/N0rtYSgR6QE8AfRR1ZNnWW0Z0EhE6olIELkdecXmjBVPiUg5Ealw+j65nfv5nlFR\nApSUz2Q6cLvr/u3A31pNIlJFRIJd90OBzkBCkSU8O09+xnn3rx/w41m+cDnN7b6ccVy/D7C+CPMV\npunAba6zoi4Cjp4+FFoonO7hd+oGJJJ7fG+V63b6zI7awOw86/UENpH7be8Zp3Pnsx/Xk/uNIh3Y\nB8w9cz/IPRNktesWXxz3w9N9KQmfiStjNWABsNn1b1XX8ljgA9f9TsBa1+eyFrjL6dwF/YyB0eR+\nuQIIAb5y/R79AdR3OvMF7MtLrt+L1cBPQFOnM59lP74A9gCZrt+Tu4DBwGDX8wKMc+3nWgo4O/J8\nbjaC2xhjjFul9jCUMcYYz1mxMMYY45YVC2OMMW5ZsTDGGOOWFQtjjDFuWbEw5hyIyPEL3H6qa7Q2\nIlJeRCaIyBbXbLuLRKSjiAS57vv0oFlTslixMKaIiEgM4K+qSa5FH5A76rmRqsaQO/tpqOZOeLcA\nGOBIUGPyYcXCmPPgGiX7qoisc10nZIBruZ+IvONqKcwUkdki0s+12SBcI7lFpAHQEXhWVXMgd8oP\nVZ3lWvc71/rGFAvWzDXm/NwAtAZaAaHAMhFZRO6UHVFAC3Jnml0PfOTapjO5o3ABYoBVqpp9ltdf\nB7T3SnJjzoO1LIw5P13Inak0W1X3AQvJ/ePeBfhKVXNUdS+500ecVgtI8eTFXUUk4/ScXsY4zYqF\nMefnbBf6KegCQKfInVMJcuciaiUiBf0OBgNp55HNmEJnxcKY87MIGCAi/iISRu4lL/8AlpB7IR0/\nEakBXJZnm/VAQwDNvWZFHDDq9BXmRKSRiPR13a8GpKhqZlHtkDEFsWJhzPn5FlhD7kylPwKPuw47\nfU3ujKDrgAnAUnKvIgcwi78Wj7vJvXhSooisJfdaCqevP3A5MNu7u2CM52zWWWMKmYiUV9XjrtbB\nH+RepXCviJQhtw+jcwEd26df4xvgKfWR66ubks/OhjKm8M0UkcpAEPC8q8WBqp4SkRHkXhd5x9k2\ndl2k5zsrFKY4sZaFMcYYt6zPwhhjjFtWLIwxxrhlxcIYY4xbViyMMca4ZcXCGGOMW1YsjDHGuPX/\nAP2wF4pSMlc1AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x5dc2bb00>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# scores_：dict with classes as the keys, and the values as the grid of scores obtained during cross-validating each fold,\n",
    "# Each dict value has shape (n_folds, len(Cs))\n",
    "n_Cs = len(Cs)\n",
    "n_classes = 3\n",
    "scores =  np.zeros((n_classes,n_Cs))\n",
    "\n",
    "for j in range(n_classes):\n",
    "        scores[j][:] = np.mean(lrcv_L1.scores_[j],axis = 0)\n",
    "    \n",
    "mse_mean = -np.mean(scores, axis = 0)\n",
    "pyplot.plot(np.log10(Cs), mse_mean.reshape(n_Cs,1)) \n",
    "#plt.plot(np.log10(reg.Cs)*np.ones(3), [0.28, 0.29, 0.30])\n",
    "pyplot.xlabel('log(C)')\n",
    "pyplot.ylabel('neg-logloss')\n",
    "pyplot.show()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 111,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[  0.00000000e+00,   0.00000000e+00,   0.00000000e+00,\n",
       "          0.00000000e+00,   0.00000000e+00,  -4.59260184e-01,\n",
       "         -3.60084269e-01,   0.00000000e+00,   0.00000000e+00,\n",
       "          0.00000000e+00,   7.27282407e-04,   0.00000000e+00,\n",
       "          9.54539107e-03,   0.00000000e+00,   2.45534621e-01,\n",
       "          6.60480053e-02,   0.00000000e+00,   0.00000000e+00,\n",
       "          0.00000000e+00,   0.00000000e+00,   0.00000000e+00,\n",
       "          0.00000000e+00,   0.00000000e+00,   0.00000000e+00,\n",
       "          0.00000000e+00,   0.00000000e+00,  -1.08066927e-02,\n",
       "          0.00000000e+00,   0.00000000e+00,   0.00000000e+00,\n",
       "          0.00000000e+00,   0.00000000e+00,   0.00000000e+00,\n",
       "          0.00000000e+00,   0.00000000e+00,   0.00000000e+00,\n",
       "          0.00000000e+00,   0.00000000e+00,   0.00000000e+00,\n",
       "          0.00000000e+00,   0.00000000e+00,   0.00000000e+00,\n",
       "          0.00000000e+00,   0.00000000e+00,   0.00000000e+00,\n",
       "          4.13954410e-02,   0.00000000e+00,   0.00000000e+00,\n",
       "          0.00000000e+00,   0.00000000e+00,   0.00000000e+00,\n",
       "          0.00000000e+00,   0.00000000e+00,   0.00000000e+00,\n",
       "          0.00000000e+00,   0.00000000e+00,   0.00000000e+00,\n",
       "          0.00000000e+00,   0.00000000e+00,   1.00117095e-01,\n",
       "          0.00000000e+00,   0.00000000e+00,   0.00000000e+00,\n",
       "          0.00000000e+00,   0.00000000e+00,   0.00000000e+00,\n",
       "          0.00000000e+00,   0.00000000e+00,   0.00000000e+00,\n",
       "          0.00000000e+00,   0.00000000e+00,   6.49033171e-02,\n",
       "          0.00000000e+00,   0.00000000e+00,   0.00000000e+00,\n",
       "          0.00000000e+00,   0.00000000e+00,   0.00000000e+00,\n",
       "          0.00000000e+00,   0.00000000e+00,   0.00000000e+00,\n",
       "          0.00000000e+00,   0.00000000e+00,   0.00000000e+00,\n",
       "          0.00000000e+00,   0.00000000e+00,   0.00000000e+00,\n",
       "          0.00000000e+00,  -1.09923902e-01,  -8.95084331e-03,\n",
       "          0.00000000e+00,   0.00000000e+00,   0.00000000e+00,\n",
       "          0.00000000e+00,   0.00000000e+00,   0.00000000e+00,\n",
       "         -2.97686662e-02,   0.00000000e+00,   1.08583939e-01,\n",
       "          0.00000000e+00,   0.00000000e+00,   0.00000000e+00,\n",
       "          0.00000000e+00,   0.00000000e+00,   0.00000000e+00,\n",
       "          0.00000000e+00,   0.00000000e+00,   0.00000000e+00,\n",
       "          0.00000000e+00,   0.00000000e+00,   0.00000000e+00,\n",
       "          0.00000000e+00,   0.00000000e+00,   0.00000000e+00,\n",
       "          0.00000000e+00,   0.00000000e+00,  -2.38977715e-02,\n",
       "          0.00000000e+00,   0.00000000e+00,   0.00000000e+00,\n",
       "          0.00000000e+00,   0.00000000e+00,   1.04550727e-01,\n",
       "          0.00000000e+00,   0.00000000e+00,   0.00000000e+00,\n",
       "          0.00000000e+00,   0.00000000e+00,   0.00000000e+00,\n",
       "          0.00000000e+00,   0.00000000e+00,   0.00000000e+00,\n",
       "          0.00000000e+00,   0.00000000e+00,   8.70081538e-04,\n",
       "          0.00000000e+00,   6.38807476e-02,   0.00000000e+00,\n",
       "          0.00000000e+00,   0.00000000e+00,   0.00000000e+00,\n",
       "          0.00000000e+00,   0.00000000e+00,   0.00000000e+00,\n",
       "          2.45113179e-02,   0.00000000e+00,   0.00000000e+00,\n",
       "          0.00000000e+00,   0.00000000e+00,   0.00000000e+00,\n",
       "          0.00000000e+00,  -4.74496622e-04,   0.00000000e+00,\n",
       "         -1.45367263e-02,   0.00000000e+00,   0.00000000e+00,\n",
       "          0.00000000e+00,   0.00000000e+00,   0.00000000e+00,\n",
       "          0.00000000e+00,   0.00000000e+00,   0.00000000e+00,\n",
       "          0.00000000e+00,   0.00000000e+00,   0.00000000e+00,\n",
       "          0.00000000e+00,   0.00000000e+00,   0.00000000e+00,\n",
       "          0.00000000e+00,   0.00000000e+00,   0.00000000e+00,\n",
       "          0.00000000e+00,   0.00000000e+00,   0.00000000e+00,\n",
       "          0.00000000e+00,   0.00000000e+00,   0.00000000e+00,\n",
       "          0.00000000e+00,   0.00000000e+00,   0.00000000e+00,\n",
       "          0.00000000e+00,   0.00000000e+00,   0.00000000e+00,\n",
       "          0.00000000e+00,   0.00000000e+00,   0.00000000e+00,\n",
       "          0.00000000e+00,   0.00000000e+00,   0.00000000e+00,\n",
       "          0.00000000e+00,   0.00000000e+00,   0.00000000e+00,\n",
       "          0.00000000e+00,   0.00000000e+00,   0.00000000e+00,\n",
       "         -4.49336020e-03,   0.00000000e+00,   0.00000000e+00,\n",
       "          0.00000000e+00,   0.00000000e+00,   0.00000000e+00,\n",
       "          0.00000000e+00,   0.00000000e+00,   0.00000000e+00,\n",
       "          0.00000000e+00,   0.00000000e+00,   0.00000000e+00,\n",
       "          0.00000000e+00,   0.00000000e+00,   0.00000000e+00,\n",
       "          0.00000000e+00,   0.00000000e+00,   0.00000000e+00,\n",
       "          4.95241254e-02,   2.29677687e-03,   0.00000000e+00,\n",
       "         -2.94565202e-02,   0.00000000e+00,   0.00000000e+00,\n",
       "          0.00000000e+00,   0.00000000e+00,   0.00000000e+00,\n",
       "          0.00000000e+00,   0.00000000e+00],\n",
       "       [  0.00000000e+00,   0.00000000e+00,  -3.88510275e-02,\n",
       "         -9.52994088e-02,  -2.24816741e-01,  -5.10689865e-01,\n",
       "         -2.94308228e-01,  -1.44488652e-01,   0.00000000e+00,\n",
       "          0.00000000e+00,   0.00000000e+00,   1.34697030e-01,\n",
       "         -3.74938109e-02,   0.00000000e+00,   1.15838206e-01,\n",
       "          0.00000000e+00,   0.00000000e+00,   0.00000000e+00,\n",
       "          0.00000000e+00,   0.00000000e+00,   0.00000000e+00,\n",
       "          0.00000000e+00,   0.00000000e+00,   0.00000000e+00,\n",
       "          0.00000000e+00,  -3.43138487e-02,   0.00000000e+00,\n",
       "          3.28911790e-02,   0.00000000e+00,   0.00000000e+00,\n",
       "          0.00000000e+00,   2.01298632e-03,   0.00000000e+00,\n",
       "          0.00000000e+00,   0.00000000e+00,   2.87866050e-02,\n",
       "          0.00000000e+00,   0.00000000e+00,   3.02122560e-03,\n",
       "         -1.22760080e-02,  -3.88867354e-03,   0.00000000e+00,\n",
       "          0.00000000e+00,   6.69815153e-02,   0.00000000e+00,\n",
       "          0.00000000e+00,   0.00000000e+00,   0.00000000e+00,\n",
       "          0.00000000e+00,   0.00000000e+00,   0.00000000e+00,\n",
       "          0.00000000e+00,   4.66099116e-02,   0.00000000e+00,\n",
       "         -1.04507468e-03,   0.00000000e+00,   0.00000000e+00,\n",
       "          0.00000000e+00,   0.00000000e+00,   0.00000000e+00,\n",
       "          0.00000000e+00,   0.00000000e+00,   4.97604301e-02,\n",
       "          0.00000000e+00,   0.00000000e+00,   0.00000000e+00,\n",
       "          0.00000000e+00,   0.00000000e+00,   0.00000000e+00,\n",
       "          0.00000000e+00,   0.00000000e+00,   0.00000000e+00,\n",
       "          0.00000000e+00,   0.00000000e+00,   6.44675513e-02,\n",
       "          0.00000000e+00,   0.00000000e+00,   0.00000000e+00,\n",
       "          0.00000000e+00,   0.00000000e+00,   0.00000000e+00,\n",
       "         -1.00301650e-05,   0.00000000e+00,   0.00000000e+00,\n",
       "          0.00000000e+00,  -1.65973690e-02,   0.00000000e+00,\n",
       "          1.17852515e-01,  -1.37043403e-02,   0.00000000e+00,\n",
       "          0.00000000e+00,   7.82431291e-02,   0.00000000e+00,\n",
       "          0.00000000e+00,   0.00000000e+00,   0.00000000e+00,\n",
       "          0.00000000e+00,   0.00000000e+00,   1.29554736e-01,\n",
       "          0.00000000e+00,  -1.57415165e-02,   0.00000000e+00,\n",
       "          1.72940293e-02,   0.00000000e+00,   5.59776707e-02,\n",
       "          0.00000000e+00,   0.00000000e+00,   0.00000000e+00,\n",
       "          0.00000000e+00,   6.39860518e-02,   0.00000000e+00,\n",
       "          0.00000000e+00,   0.00000000e+00,   0.00000000e+00,\n",
       "          1.54862309e-01,   0.00000000e+00,   0.00000000e+00,\n",
       "          0.00000000e+00,   0.00000000e+00,   0.00000000e+00,\n",
       "         -1.01685004e-01,   0.00000000e+00,   0.00000000e+00,\n",
       "          0.00000000e+00,   0.00000000e+00,   0.00000000e+00,\n",
       "          0.00000000e+00,   0.00000000e+00,   0.00000000e+00,\n",
       "          0.00000000e+00,   0.00000000e+00,   0.00000000e+00,\n",
       "          0.00000000e+00,   0.00000000e+00,   0.00000000e+00,\n",
       "          0.00000000e+00,  -1.86746449e-02,   0.00000000e+00,\n",
       "          0.00000000e+00,   0.00000000e+00,   0.00000000e+00,\n",
       "          0.00000000e+00,   0.00000000e+00,   0.00000000e+00,\n",
       "          6.00654978e-02,   0.00000000e+00,   6.13847665e-03,\n",
       "          0.00000000e+00,   0.00000000e+00,   0.00000000e+00,\n",
       "          0.00000000e+00,   0.00000000e+00,  -1.62355621e-01,\n",
       "          2.20479746e-02,  -7.41777530e-03,   0.00000000e+00,\n",
       "         -1.20208434e-04,   8.36161373e-02,   0.00000000e+00,\n",
       "          0.00000000e+00,   0.00000000e+00,   0.00000000e+00,\n",
       "          0.00000000e+00,   0.00000000e+00,   0.00000000e+00,\n",
       "          0.00000000e+00,   0.00000000e+00,   0.00000000e+00,\n",
       "          0.00000000e+00,   0.00000000e+00,   0.00000000e+00,\n",
       "          0.00000000e+00,  -1.10710511e-04,   5.82900665e-02,\n",
       "         -1.38940330e-03,   0.00000000e+00,   0.00000000e+00,\n",
       "          0.00000000e+00,   0.00000000e+00,   1.87943397e-02,\n",
       "          0.00000000e+00,   0.00000000e+00,   0.00000000e+00,\n",
       "          0.00000000e+00,   0.00000000e+00,   0.00000000e+00,\n",
       "         -1.04817812e-03,   0.00000000e+00,   0.00000000e+00,\n",
       "          0.00000000e+00,   0.00000000e+00,   0.00000000e+00,\n",
       "          0.00000000e+00,   0.00000000e+00,   0.00000000e+00,\n",
       "          0.00000000e+00,   0.00000000e+00,   0.00000000e+00,\n",
       "          0.00000000e+00,   1.19316005e-02,   0.00000000e+00,\n",
       "          0.00000000e+00,   0.00000000e+00,   0.00000000e+00,\n",
       "          0.00000000e+00,   0.00000000e+00,  -3.13891537e-02,\n",
       "          4.00996388e-02,   5.44990964e-02,   0.00000000e+00,\n",
       "          0.00000000e+00,   0.00000000e+00,  -2.49313136e-02,\n",
       "          0.00000000e+00,   0.00000000e+00,   0.00000000e+00,\n",
       "          0.00000000e+00,   0.00000000e+00,   0.00000000e+00,\n",
       "          0.00000000e+00,   0.00000000e+00,   0.00000000e+00,\n",
       "          0.00000000e+00,   0.00000000e+00],\n",
       "       [  0.00000000e+00,   0.00000000e+00,   2.48966749e-02,\n",
       "          1.15294279e-01,   2.46569135e-01,   9.52593672e-01,\n",
       "          2.84101848e-01,   3.43218632e-01,   0.00000000e+00,\n",
       "          0.00000000e+00,   0.00000000e+00,  -7.82176006e-02,\n",
       "          0.00000000e+00,   0.00000000e+00,  -2.99165600e-01,\n",
       "          0.00000000e+00,  -3.18396957e-03,   0.00000000e+00,\n",
       "          0.00000000e+00,   0.00000000e+00,   0.00000000e+00,\n",
       "         -7.41496719e-04,  -2.92033797e-03,  -6.54745653e-05,\n",
       "          1.03756873e-03,   2.70021655e-02,   0.00000000e+00,\n",
       "         -8.12656538e-04,   0.00000000e+00,   0.00000000e+00,\n",
       "          0.00000000e+00,   0.00000000e+00,   0.00000000e+00,\n",
       "          0.00000000e+00,   0.00000000e+00,   0.00000000e+00,\n",
       "          0.00000000e+00,   0.00000000e+00,   0.00000000e+00,\n",
       "          6.65552875e-02,   2.87820578e-02,   0.00000000e+00,\n",
       "          0.00000000e+00,  -5.75881641e-02,   0.00000000e+00,\n",
       "          0.00000000e+00,   0.00000000e+00,   0.00000000e+00,\n",
       "          0.00000000e+00,   0.00000000e+00,   0.00000000e+00,\n",
       "          0.00000000e+00,   0.00000000e+00,   0.00000000e+00,\n",
       "          1.47577856e-02,   0.00000000e+00,   0.00000000e+00,\n",
       "          0.00000000e+00,   0.00000000e+00,  -4.62830861e-02,\n",
       "          0.00000000e+00,   0.00000000e+00,  -1.56832724e-02,\n",
       "          8.25750180e-02,   0.00000000e+00,   0.00000000e+00,\n",
       "          0.00000000e+00,   0.00000000e+00,   0.00000000e+00,\n",
       "          0.00000000e+00,   1.83691895e-04,  -2.04392132e-02,\n",
       "          0.00000000e+00,   0.00000000e+00,  -1.01594323e-01,\n",
       "          0.00000000e+00,   0.00000000e+00,   0.00000000e+00,\n",
       "          0.00000000e+00,   0.00000000e+00,   0.00000000e+00,\n",
       "          3.41057727e-02,   0.00000000e+00,   0.00000000e+00,\n",
       "          0.00000000e+00,   2.16608081e-02,   0.00000000e+00,\n",
       "         -1.15681430e-01,   9.31290462e-02,   0.00000000e+00,\n",
       "          2.97935945e-03,  -5.79991905e-02,   0.00000000e+00,\n",
       "          2.10704670e-05,   0.00000000e+00,   0.00000000e+00,\n",
       "          4.80434702e-02,   0.00000000e+00,  -2.22525111e-01,\n",
       "          0.00000000e+00,   5.16588465e-02,   0.00000000e+00,\n",
       "          0.00000000e+00,   0.00000000e+00,  -4.39687193e-02,\n",
       "          0.00000000e+00,   0.00000000e+00,   0.00000000e+00,\n",
       "          0.00000000e+00,  -6.14799999e-02,   0.00000000e+00,\n",
       "          0.00000000e+00,   0.00000000e+00,   0.00000000e+00,\n",
       "         -1.94287121e-01,   0.00000000e+00,   0.00000000e+00,\n",
       "          0.00000000e+00,   0.00000000e+00,   0.00000000e+00,\n",
       "          1.06018591e-01,   0.00000000e+00,  -8.99400369e-03,\n",
       "          0.00000000e+00,   0.00000000e+00,   0.00000000e+00,\n",
       "          0.00000000e+00,   0.00000000e+00,   0.00000000e+00,\n",
       "          0.00000000e+00,   0.00000000e+00,   0.00000000e+00,\n",
       "          2.40431377e-02,   0.00000000e+00,   0.00000000e+00,\n",
       "          0.00000000e+00,   0.00000000e+00,   0.00000000e+00,\n",
       "          2.66479443e-02,   0.00000000e+00,   0.00000000e+00,\n",
       "          0.00000000e+00,   0.00000000e+00,   0.00000000e+00,\n",
       "         -1.22955052e-01,   0.00000000e+00,   0.00000000e+00,\n",
       "          0.00000000e+00,   0.00000000e+00,   0.00000000e+00,\n",
       "          0.00000000e+00,   4.85950038e-02,   1.71829508e-01,\n",
       "          0.00000000e+00,   0.00000000e+00,   0.00000000e+00,\n",
       "          5.99402518e-04,  -1.10984226e-01,   0.00000000e+00,\n",
       "          0.00000000e+00,   0.00000000e+00,   0.00000000e+00,\n",
       "          0.00000000e+00,   0.00000000e+00,   0.00000000e+00,\n",
       "          0.00000000e+00,   0.00000000e+00,   0.00000000e+00,\n",
       "          0.00000000e+00,   0.00000000e+00,   0.00000000e+00,\n",
       "          0.00000000e+00,   3.63833466e-02,  -1.42818811e-02,\n",
       "          5.06476818e-02,   0.00000000e+00,   0.00000000e+00,\n",
       "          0.00000000e+00,   0.00000000e+00,  -6.36522824e-02,\n",
       "          0.00000000e+00,   0.00000000e+00,   0.00000000e+00,\n",
       "          0.00000000e+00,   0.00000000e+00,   4.18212917e-03,\n",
       "          1.90792803e-03,   0.00000000e+00,   0.00000000e+00,\n",
       "          0.00000000e+00,   0.00000000e+00,   0.00000000e+00,\n",
       "          0.00000000e+00,   0.00000000e+00,   1.16545912e-05,\n",
       "          0.00000000e+00,   0.00000000e+00,   0.00000000e+00,\n",
       "          0.00000000e+00,  -4.64582263e-02,   0.00000000e+00,\n",
       "          0.00000000e+00,   0.00000000e+00,   0.00000000e+00,\n",
       "          9.18980607e-06,   0.00000000e+00,   2.31637331e-02,\n",
       "         -5.98742648e-03,  -8.84174342e-02,   2.82723303e-02,\n",
       "          0.00000000e+00,   0.00000000e+00,   4.22850081e-02,\n",
       "          0.00000000e+00,  -9.37115220e-02,   0.00000000e+00,\n",
       "          1.02183399e-03,   1.05479385e-02,   0.00000000e+00,\n",
       "          0.00000000e+00,   0.00000000e+00,   0.00000000e+00,\n",
       "          0.00000000e+00,   0.00000000e+00]])"
      ]
     },
     "execution_count": 111,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lrcv_L1.coef_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 125,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "y_pred = lrcv_L1.predict(X_test)\n",
    "y_pred_l1 = pd.DataFrame(y_pred,index=test_id,columns=['interest_level'])\n",
    "y_pred_l1.to_csv(\"l1_pred.csv\",header=True,index_label='Id')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## L2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 112,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "LogisticRegressionCV(Cs=[0.01, 0.1, 1, 10], class_weight=None, cv=5,\n",
       "           dual=False, fit_intercept=True, intercept_scaling=1.0,\n",
       "           max_iter=100, multi_class='ovr', n_jobs=1, penalty='l2',\n",
       "           random_state=None, refit=True, scoring='neg_log_loss',\n",
       "           solver='liblinear', tol=0.0001, verbose=0)"
      ]
     },
     "execution_count": 112,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lr_cv_L2 = LogisticRegressionCV(Cs=Cs, cv = 5, scoring='neg_log_loss', penalty='l2', solver='liblinear', multi_class='ovr')\n",
    "lr_cv_L2.fit(X_train, y_train) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 124,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_pred = lr_cv_L2.predict(X_test)\n",
    "y_pred_l2 = pd.DataFrame(y_pred,index=test_id,columns=['interest_level'])\n",
    "y_pred_l2.to_csv(\"l2_pred.csv\",header=True,index_label='Id')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 117,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYsAAAEKCAYAAADjDHn2AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4wLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvpW3flQAAIABJREFUeJzt3Xl8VOXZ//HPRdgUkEUQkUUQQRRB\nwBCiqLVWW2pbsGplE4gCsbVora1W7fao9ddWu1d8KksUkc3iAi4tVYva+kBI2HeIgBLWAAICsiS5\nfn/MpKYxYQbIyZlMvu/Xa16Zc859Zq6TSeY7Z5n7NndHRETkeGqFXYCIiCQ+hYWIiMSksBARkZgU\nFiIiEpPCQkREYlJYiIhITAoLERGJSWEhIiIxKSxERCSm2mEXUFmaN2/u7du3D7sMEZFqZeHChbvc\nvUWsdkkTFu3btyc3NzfsMkREqhUz+zCedjoMJSIiMSksREQkJoWFiIjEpLAQEZGYFBYiIhKTwkJE\nRGIKNCzMrJ+ZrTWzPDN7oII2t5jZKjNbaWZTyyw7w8y2mNmTQdYpIiLHF9j3LMwsBRgLXAfkAzlm\nNtvdV5Vq0wl4EOjr7h+b2VllHuZR4N2gahQRqe7mrNzOoaOFfLNnm0CfJ8g9izQgz903uPtRYDow\noEyb0cBYd/8YwN13liwws0uBlsA/AqxRRKTa+tvybXx3yiKen/8RRcUe6HMFGRatgc2lpvOj80rr\nDHQ2s/fNbL6Z9QMws1rAb4H7jvcEZpZpZrlmlltQUFCJpYuIJLZXl25lzLTFXNK2Cc/e1puUWhbo\n8wUZFuVVXjb6agOdgKuBwcAEM2sC3Am84e6bOQ53H+fuqe6e2qJFzK5NRESSwqwlW/je9MX0ateE\nSben0ah+ncCfM8i+ofKBtqWm2wBby2kz392PARvNbC2R8LgMuNLM7gQaAnXN7IC7l3uSXESkpnhx\nYT73zVxKWodmTBzRmwb1qqaLvyD3LHKATmbWwczqAoOA2WXavAJ8EcDMmhM5LLXB3Ye6ezt3bw/8\nEHhOQSEiNd0LOZv54cylXNbxTJ7JSKuyoIAAw8LdC4ExwBxgNfCCu680s0fMrH+02Rxgt5mtAuYC\n97n77qBqEhGprqZmf8T9Ly7jivObM3FEb06rm1Klz2/uwZ5BryqpqamuLspFJBlNnv8hP31lBVdf\n0IK/3Hop9etUXlCY2UJ3T43VLmnGsxARSUbPvr+R/3l1FV/qchZP3dqLerWrdo+ihMJCRCRBTfjX\nBn7x+mq+fFFLnhzSi7q1w+uhSWEhIpKA/vLuB/zqb2v46sVn86fBPamTEm5XfgoLEZEEM3ZuHk/M\nWcvXu7fi9wN7hB4UoLAQEUkof3xrPb9/ax0DepzDb791CbUTIChAYSEikhDcnd+/uY4//TOPG3u1\n5ombLwm8C48TobAQEQmZu/PEnLU89c4H3JLahl/e2D2hggIUFiIioXJ3fvW3NTz93gYGp7XjsRsu\nplaCBQUoLEREQuPuPPraarLe38iw9HN5uH/XhAwKUFiIiITC3fmf2SuZNO9DMi5vz8+/cRFmiRkU\noLAQEalyxcXOz2av4Pn5HzHqig78+GsXJnRQgMJCRKRKFRc7D728nOk5m7njC+fxQL8uCR8UoLAQ\nEakyRcXOj15cxsyF+Yz54vn84Mudq0VQgMJCRKRKFBU79/11KS8t3sL3vtSJe67tVG2CAhQWIiKB\nKywq5t4XljJ76Vbuva4zd3+pU9glnTCFhYhIgI4VFXPP9CW8vnwb9/e7gDuvPj/skk6KwkJEJCBH\nC4u5e9pi/r5yOw9d34XMqzqGXdJJU1iIiATgSGERY6Yu5s1VO/jp1y9i5BUdwi7plCgsREQq2eFj\nRdw5ZRH/XLOTh/t3ZcTl7cMu6ZQpLEREKtHhY0XcMXkh764r4Bc3XMyt6eeGXVKlUFiIiFSST48W\nkTk5l3/n7eJXN3ZjUFq7sEuqNAoLEZFKcOhoIaMm5TJvw24ev6k730ptG3ZJlUphISJyig4eKeT2\nZ3PI2bSH391yCd/s2SbskipdoOP1mVk/M1trZnlm9kAFbW4xs1VmttLMpkbn9TCzedF5y8xsYJB1\nioicrANHCsl4ZgE5m/bw+4E9kjIoIMA9CzNLAcYC1wH5QI6ZzXb3VaXadAIeBPq6+8dmdlZ00SFg\nuLuvN7NzgIVmNsfd9wZVr4jIidp/+BgZWQtYmr+PPw/uxde6twq7pMAEuWeRBuS5+wZ3PwpMBwaU\naTMaGOvuHwO4+87oz3Xuvj56fyuwE2gRYK0iIidk36fHGDZxAcvy9zF2SM+kDgoINixaA5tLTedH\n55XWGehsZu+b2Xwz61f2QcwsDagLfBBYpSIiJ2DvoaPcOiGbVVv38dTQXvS7OLmDAoI9wV1ed4pe\nzvN3Aq4G2gD/MrOLSw43mVkrYDIwwt2LP/cEZplAJkC7dslziZqIJK6PDx5l6IRs8nYe4Olhl3JN\nl5Zhl1QlgtyzyAdKXzvWBthaTptZ7n7M3TcCa4mEB2Z2BvA68BN3n1/eE7j7OHdPdffUFi10lEpE\ngrX7wBEGj59PXsEBxg2vOUEBwYZFDtDJzDqYWV1gEDC7TJtXgC8CmFlzIoelNkTbvww85+5/DbBG\nEZG4FHwSCYqNuw4ycUQqV19wVuyVkkhgYeHuhcAYYA6wGnjB3Vea2SNm1j/abA6w28xWAXOB+9x9\nN3ALcBWQYWZLorceQdUqInI8O/cfZtC4eWze8ynPZPTmyk4170iGuZc9jVA9paamem5ubthliEiS\n2b7vMEPGz2f7/sNkZfQm/bwzwy6pUpnZQndPjdVO3+AWEanA1r2fMnj8fHZ9coRJt6fRu32zsEsK\njcJCRKQc+R8fYvD4+ew9eIznRvbh0nObhl1SqBQWIiJlbN5ziEHj5rP/8DEmj+pDj7ZNwi4pdAoL\nEZFSPtx9kMHj5nPwaBFTR6XTrU3jsEtKCAoLEZGojbsiQXGksIipo/vQ9RwFRQmFhYgIkLfzAEPG\nz6ew2Jk6Op0LW50RdkkJRWEhIjXe+h2fMHh8NuBMG53OBWc3CrukhKOwEJEabe32Txgyfj61ahnT\nRqdz/lkKivIEOviRiEgiW7V1P4PGzaN2ijE9U0FxPNqzEJEaacWWfdw6MZvT6qQwbXQ67Zs3CLuk\nhKY9CxGpcZbl72XI+Pk0qFubGZmXKSjioD0LEalRFn/0McOzFtDk9DpMHZVO22anh11StaCwEJEa\nY+GHexiRlcOZDesydXQ6rZucFnZJ1YYOQ4lIjbBg4x6GT1xAi0b1mJ6poDhRCgsRSXrzPtjNiKwF\nnN24PtMz02nVWEFxohQWIpLU3s/bxW3PLqBN09OYlplOyzPqh11StaRzFiKStN5dV0Dmc7m0P7MB\nU0b3oXnDemGXVG0pLEQkKc1ds5M7nl9IxxYNmTKqD80a1A27pGpNYSEiSeetVTu4c8oiOp/dkOdH\n9qHJ6QqKU6VzFiKSVP6+YjvfmbKQLq0aMWVkuoKikigsRCRpvLF8G2OmLqLrOY2ZPLIPjU+vE3ZJ\nSUOHoUQkKby6dCv3zFhCj7ZNePa23jSqr6CoTNqzEJFq75XFW/je9MVc2q4pk25PU1AEQHsWIlKt\nzVyYz30zl5Le4UwmZqRyel29rQUh0D0LM+tnZmvNLM/MHqigzS1mtsrMVprZ1FLzR5jZ+uhtRJB1\nikj19ELOZu6buZS+HZuTldFbQRGgwH6zZpYCjAWuA/KBHDOb7e6rSrXpBDwI9HX3j83srOj8ZsDP\ngVTAgYXRdT8Oql4RqV6mZn/EQy8v56rOLRg37FLq10kJu6SkFuSeRRqQ5+4b3P0oMB0YUKbNaGBs\nSQi4+87o/K8Ab7r7nuiyN4F+AdYqItXI5HmbeOjl5XzxAgVFVQkyLFoDm0tN50fnldYZ6Gxm75vZ\nfDPrdwLrYmaZZpZrZrkFBQWVWLqIJKpn3t/IT2et5NoLz+IvCooqE2RYWDnzvMx0baATcDUwGJhg\nZk3iXBd3H+fuqe6e2qJFi1MsV0QS3YR/beDhV1fxla4teWropdSrraCoKkGGRT7QttR0G2BrOW1m\nufsxd98IrCUSHvGsKyI1yF/e/YBfvL6a67udzZNDelG3tq78r0pB/rZzgE5m1sHM6gKDgNll2rwC\nfBHAzJoTOSy1AZgDfNnMmppZU+DL0XkiUgONnZvHr/62hm9ccg5/GtSTOikKiqoW2NVQ7l5oZmOI\nvMmnAFnuvtLMHgFy3X02n4XCKqAIuM/ddwOY2aNEAgfgEXffE1StIpK4/vjWen7/1jpu6HEOv/nW\nJdRWUITC3D93KqBaSk1N9dzc3LDLEJFK4u78/s11/OmfedzUqw2P39ydlFrlnc6UU2FmC909NVY7\nfYNFRBKOu/PEnLU89c4HDExtyy9v7EYtBUWo4tqfM7O+ZtYgev9WM/udmZ0bbGkiUhO5O7/82xqe\neucDhvRpp6BIEPEe/Ptf4JCZXQLcD3wIPBdYVSJSI7k7j762mnHvbWD4Zefy2A0XKygSRLxhUeiR\nkxsDgD+6+x+BRsGVJSI1jbvzP7NXkvX+Rm7r256H+3fFTEGRKOI9Z/GJmT0I3ApcFe33SX0Ai0il\nKC52fjprBVOyP2L0lR146PoLFRQJJt49i4HAEWCku28n0vXGE4FVJSI1RnGx89DLy5mS/RHf/kJH\nBUWCinvPgsjhpyIz6wx0AaYFV5aI1ARFxc6PXlzGzIX53HXN+dx7XWcFRYKKd8/iPaCembUG3gZu\nA54NqigRSX5Fxc4P/7qUmQvzuefaTvzgyxcoKBJYvGFh7n4IuBH4s7t/E+gaXFkikswKi4r5/owl\nvLx4Cz+4rjP3XNs57JIkhrjDwswuA4YCr0fnqbtHETlhx4qK+d70JcxeupUf9evCXV/qFHZJEod4\nz1ncQ2REu5ej/TudB8wNriwRSUZHC4u5a9oi5qzcwY+vv5DRV50XdkkSp7jCwt3fBd41s0Zm1tDd\nNwB3B1uaiCSTI4VFfHfKYt5avYOfff0ibr+iQ9glyQmIt7uPbma2GFgBrDKzhWamcxYiEpfDx4r4\nzvOLeGv1Dh4Z0FVBUQ3FexjqaeBed58LYGZXA+OBywOqS0SSxOFjRdwxeSHvrivgsW9ezNA+6lau\nOoo3LBqUBAWAu79T0rGgiEhFPj1aRObkXP6dt4tf39SNgb3bhV2SnKR4w2KDmf0UmBydvhXYGExJ\nIpIMDh0tZOSzuczfuJsnbr6Emy9tE3ZJcgrivXT2dqAF8BLwcvT+bUEVJSLV28EjhWQ8k0P2xt38\n7hYFRTKI92qoj9HVTyIShwNHCsnIWsDizXv5w6Ce9L/knLBLkkpw3LAws1eBCsdddff+lV6RiFRb\n+w8fY0TWApbn7+PPg3tyfbdWYZcklSTWnsVvqqQKEan29n16jOETs1m5dT9PDulFv4vPDrskqUTH\nDYvol/FERI5r76GjDJu4gDXb9/O/t17KdRe1DLskqWRxnbMws+V8/nDUPiAX+IW7767swkSkethz\n8Ci3Tsgmr+AA44al8sUuZ4VdkgQg3ktn/wYUAVOj04MAIxIYzwLfqPTKRCTh7T5whKETstmw6yDj\nh6fyhc4twi5JAhLvpbN93f1Bd18evf0Y+IK7/xpoX9FKZtbPzNaaWZ6ZPVDO8gwzKzCzJdHbqFLL\nHjezlWa22sz+ZOroXiShFHxyhMHj57Np90GyRvRWUCS5eMOioZn1KZkwszSgYXSysLwVouN0jwW+\nClwEDDazi8ppOsPde0RvE6LrXg70BboDFwO9gS/EWauIBGzn/sMMGjePzXs+JSujN1d0ah52SRKw\neA9DjQKyzKwhkcNP+4GR0S4/flnBOmlAXrSHWsxsOjAAWBXH8zlQH6gbfb46wI44axWRAG3fd5gh\n4+ezff9hnr2tN33OOzPskqQKxPulvBygm5k1JjJq3t5Si1+oYLXWwOZS0/lAn3La3WRmVwHrgO+7\n+2Z3n2dmc4FtRMLiSXdfHU+tIhKcrXs/ZfD4+ew+cJTnbk8jtX2zsEuSKhJvF+WNzex3RMbffsvM\nfhsNjuOuVs68sldUvQq0d/fuwFvApOjznQ9cCLQhEjrXRAOlbF2ZZpZrZrkFBQXxbIqInKTNew4x\ncNw89hw4ynMjFRQ1TbznLLKAT4Bborf9wDMx1skH2paabgNsLd3A3Xe7+5Ho5Hjg0uj9bwLz3f2A\nux8gcjVWetkncPdx7p7q7qktWujkmkhQNu46yMCn57Hv0DGeH9WHXu2ahl2SVLF4w6Kju//c3TdE\nbw8DscZDzAE6mVkHM6tL5HLb2aUbmFnpvgD6AyWHmj4CvmBmtc2sDpGT2zoMJRKC9Ts+4Zan53G4\nsJhpmelc0rZJ2CVJCOI9wf2pmV3h7v8GMLO+wKfHW8HdC81sDDAHSAGyouN3PwLkuvts4G4z60/k\niqo9QEZ09ZnANUDJlwH/7u6vntimicipWrl1H8MmLiClljEjM51OLRuFXZKExNwr7Cfws0ZmPYic\nT2hM5FzEHiDD3ZcGW178UlNTPTc3N+wyRJLGks17GT4xm4b1ajNldDodmmu8s2RkZgvdPTVWu3iv\nhloCXGJmZ0Sn959ifSKSwHI27eG2Z3Jo1qAuU0b1oW2z08MuSUIWq4vyeyuYD4C7/y6AmkQkRO/n\n7WLUpFxaNanP1FHpnN24ftglSQKItWehA5QiNcjcNTu54/mFnNe8AZNH9qFFo3phlyQJIlYX5Q9X\nVSEiEq6/r9jOXdMWccHZjZh8ex+aNqgbdkmSQOK9dPY/zGxREIWISHhmLdnCd6cuolvrxkwZla6g\nkM+J99LZ0tT7q0gSeSFnMz96aRl9OjRj4ojeNKh3Mm8LkuxO5q/i9UqvQkRC8dy8Tfxs1kqu7NSc\nccNSOa1uStglSYI64bBw958EUYiIVK3x723gsTdWc+2FLRk7tCf1aisopGLxDqv6CRUPq/qDkm7I\nRSTxuTt//mcev3tzHV/r1oo/DOpBnZQTPn0pNUy8exa/I9IJ4FQi5ywGAWcDa4l0Mnh1EMWJSOVy\nd56Ys5an3vmAG3u15vGbulNbQSFxiPevpJ+7P+3un7j7fncfB1zv7jMAdT8pUg24O4++tpqn3vmA\nIX3a8ZubL1FQSNzi/UspNrNbzKxW9HZLqWWxO5cSkVAVFzs/fmUFWe9v5La+7XnshoupVUsXNkr8\n4g2LocAwYCeR4U2HAbea2WnAmIBqE5FKUFhUzA9nLmVq9kfceXVHfvb1i/7TZY9IvOLtSHAD8I0K\nFv+78soRkcp0rKiYe2Ys4fVl27j3us7cdc35Cgo5KfEOq9rZzN42sxXR6e5mpktoRRLYkcIivvP8\nIl5fto2Hru/C3V/qpKCQkxbvYajxwIPAMQB3X0bkiigRSUCfHi1i1KRc3lq9g0cGdCXzqo5hlyTV\nXLyXzp7u7gvKfCopDKAeETlFB48UMnJSDtkb9/D4Td25pXfbsEuSJBBvWOwys45Er3wys5uBbYFV\nJSInZf/hY2RkLWBp/j7+MLAHA3q0DrskSRLxhsV3gXFAFzPbAmwkcoWUiCSIjw8eZXjWAtZs38/Y\nIT3pd3GrsEuSJBJvWGwBngHmAs2A/cAI4JGA6hKRE1DwyRGGTcxmw66DPD3sUq7p0jLskiTJxBsW\ns4C9wCIi3X6ISILYvu8wQybMZ9vewzyT0Zu+5zcPuyRJQvGGRRt37xdoJSJywjbvOcTQCdnsOXiU\nSbenkdahWdglSZKK99LZ/zOzboFWIiInZOOugwx8eh57Dx3l+VF9FBQSqHj3LK4AMsxsI3CESM+z\n7u7dA6tMRCq0fscnDJ2QTWGxMy0zna7nNA67JEly8YbFV0/mwc2sH/BHIAWY4O6/KrM8A3iCyAl0\ngCfdfUJ0WTtgAtCWyCW717v7ppOpQySZrNy6j2ETF5BSy5iRmU6nlo3CLklqgHj7hvrwRB/YzFKA\nscB1QD6QY2az3X1VmaYz3L28zgifAx5z9zfNrCFQfKI1iCSbJZv3MnxiNg3r1WbK6HQ6NG8QdklS\nQwTZmX0akOfuG9z9KDAdGBDPimZ2EVDb3d8EcPcD7n4ouFJFEl/Opj3cOiGbxqfXYcYdlykopEoF\nGRatgc2lpvOj88q6ycyWmdlMMyvpl6AzsNfMXjKzxWb2RHRP5b+YWaaZ5ZpZbkFBQeVvgUiCeD9v\nF8MnLuCsM+rx1zsup22z08MuSWqYIMOivO4tyw6U9CrQPnqi/C1gUnR+beBK4IdAb+A8IONzD+Y+\nzt1T3T21RYsWlVW3SEKZu2Yntz2bw7lnns6MzMs4u3H9sEuSGijIsMgncnK6RBvKfKHP3Xe7+5Ho\n5Hjg0lLrLo4ewioEXgF6BVirSEL6+4rtZE7OpXPLhkwbnU6LRvXCLklqqCDDIgfoZGYdzKwukS7N\nZ5duYGalO6/pD6wutW5TMyvZXbgGKHtiXCSpzVqyhe9OXUS31o2ZMiqdpg3qhl2S1GDxXjp7wty9\n0MzGAHOIXDqb5e4rzewRINfdZwN3m1l/It2d7yF6qMndi8zsh8DbFukXfSGRPQ+RGuGFnM386KVl\npLVvxsSM3jSsF9i/qkhczL3saYTqKTU11XNzc8MuQ+SUPTdvEz+btZIrOzVn3LBUTqv7uWs7RCqN\nmS1099RY7fRxRSSBjH9vA4+9sZprL2zJ2KE9qVdbQSGJQWEhkiD+/PZ6fvvmOr7WrRV/GNSDOilB\nnlIUOTEKC5GQuTu/+cdaxs79gBt7tebxm7pTW0EhCUZhIRIid+fR11aT9f5GBqe147EbLqZWrfK+\noiQSLoWFSEiKi52fzFrB1OyPyLi8PT//xkVELv4TSTwKC5EQFBYVc/+Ly3hp0Ra+c3VH7v/KBQoK\nSWgKC5EqdqyomHtmLOH1Zdu497rO3HXN+QoKSXgKC5EqdKSwiDFTF/Pmqh08dH0XMq/qGHZJInFR\nWIhUkU+PFnHH8wt5b10BjwzoyvDL2oddkkjcFBYiVeDgkUJGTsohe+MeHr+pO7f0bht7JZEEorAQ\nCdj+w8fIyFrA0vx9/GFgDwb0KG9YF5HEprAQCdDHB48yPGsBa7bv58nBPflqt1axVxJJQAoLkYAU\nfHKEYROz2bDrIE8Pu5RrurQMuySRk6awEAnA9n2HGTJhPtv2HuaZjN70Pb952CWJnBKFhUgly//4\nEEPGZ7Pn4FEm3Z5GWodmYZckcsoUFiKVaNOugwwZP58DRwp5flQferRtEnZJIpVCYSFSSdbv+ISh\nE7IpLHamZabT9ZzGYZckUmkUFiKVYOXWfQybuICUWsb0zHQ6t2wUdkkilUqd5oucoiWb9zJ43Hzq\n167FC3dcpqCQpKQ9C5FTkLNpD7c9k0PTBnWYOiqdts1OD7skkUAoLERO0v/l7WLkpFxaNanP1FHp\nnN24ftgliQRGh6FETsLcNTvJeDaHc888nRmZlykoJOlpz0LkBP19xXbumraIC85uxOTb+9C0Qd2w\nSxIJXKB7FmbWz8zWmlmemT1QzvIMMyswsyXR26gyy88wsy1m9mSQdYrEa9aSLXx36iIubt2YKaPS\nFRRSYwS2Z2FmKcBY4DogH8gxs9nuvqpM0xnuPqaCh3kUeDeoGkVOxAs5m/nRS8tIa9+MiRm9aVhP\nO+ZScwS5Z5EG5Ln7Bnc/CkwHBsS7spldCrQE/hFQfSJxe27eJu5/cRlXnN+cZ29LU1BIjRNkWLQG\nNpeazo/OK+smM1tmZjPNrC2AmdUCfgvcF2B9InEZ/94GfjZrJdde2JIJI1I5rW5K2CWJVLkgw6K8\nEei9zPSrQHt37w68BUyKzr8TeMPdN3McZpZpZrlmlltQUHDKBYuU9ee31/PYG6v5WrdW/O+tvahX\nW0EhNVOQ+9L5QOmxI9sAW0s3cPfdpSbHA7+O3r8MuNLM7gQaAnXN7IC7P1Bm/XHAOIDU1NSyQSRy\n0tyd3/xjLWPnfsCNPVvz+M3dqZ2iK82l5goyLHKATmbWAdgCDAKGlG5gZq3cfVt0sj+wGsDdh5Zq\nkwGklg0KkaC4O4++tpqs9zcyOK0dj91wMbVqlbejLFJzBBYW7l5oZmOAOUAKkOXuK83sESDX3WcD\nd5tZf6AQ2ANkBFWPSDyKi52fzFrB1OyPyLi8PT//xkWYKShEzD05jt6kpqZ6bm5u2GVINVZYVMz9\nLy7jpUVb+M7VHbn/KxcoKCTpmdlCd0+N1U7X/4kAx4qKuWfGEl5fto17r+vMXdecr6AQKUVhITXe\nkcIixkxdzJurdvDQ9V3IvKpj2CWJJByFhdRonx4t4o7nF/LeugIeGdCV4Ze1D7skkYSksJAa6+CR\nQkZOyiF74x5+fVM3BvZuF3ZJIglLYSE10v7Dx8jIWsDS/H38YWAPBvQor3MBESmhsJAa5+ODRxme\ntYA12/fz5OCefLVbq7BLEkl4CgupUQo+OcKwidls2HWQp4ddyjVdWoZdkki1oLCQGmP7vsMMnTCf\nrXsP80xGb/qe3zzskkSqDYWF1Aj5Hx9iyPhs9hw8yqTb00jr0CzskkSqFYWFJL1Nuw4yZPx8Dhwp\nZPLINHq2axp2SSLVjsJCktr6HZ8wdEI2hcXOtMx0up7TOOySRKolhYUkrZVb9zFs4gJSahnTM9Pp\n3LJR2CWJVFvqoF+S0tLNexk8bj71a9fihTsuU1CInCLtWUjSydm0h9ueyaFpgzpMHZVO22anh12S\nSLWnsJCk8n95uxg5KZdWjeszdXQ6ZzeuH3ZJIklBh6Ekacxds5OMZ3No1+x0ZtxxmYJCpBJpz0KS\nwt9XbOeuaYu44OxGPHd7H5o1qBt2SSJJRWEh1d6sJVu494WldG/TmGdvS6PxaXXCLkkk6egwlFRr\nL+Ru5p4ZS0g9tymTR/ZRUIgERHsWUm1NnreJn85ayZWdmjNuWCqn1U0JuySRpKWwkGppwr828IvX\nV3PthWcxdmgv6tVWUIgESWEh1c6f317Pb99cx9e6teL3A3tQt7aOpooETWEh1Ya785t/rGXs3A+4\nsWdrHr+5O7VTFBQiVUFhIdWCu/Poa6vJen8jg9Pa8dgNF1OrloVdlkiNEejHMjPrZ2ZrzSzPzB4o\nZ3mGmRWY2ZLobVR0fg8zm2d9Oa1XAAAI7klEQVRmK81smZkNDLJOSWzFxc6PX1lB1vsbybi8Pf/v\nmwoKkaoW2J6FmaUAY4HrgHwgx8xmu/uqMk1nuPuYMvMOAcPdfb2ZnQMsNLM57r43qHolMRUVO/fP\nXMaLi/L5ztUduf8rF2CmoBCpakEehkoD8tx9A4CZTQcGAGXD4nPcfV2p+1vNbCfQAlBY1CDHior5\n/owlvLZsG/de15m7rjlfQSESkiDDojWwudR0PtCnnHY3mdlVwDrg++5eeh3MLA2oC3xQdkUzywQy\nAdq1a1dJZcuJcPfoz+h0qfmf3S9Z9t9tKWd5yf3CIueHM5fy5qodPPjVLtzxhY7BbICIxCXIsCjv\nI2CZtwleBaa5+xEz+zYwCbjmPw9g1gqYDIxw9+LPPZj7OGAcQGpqatnHjsveQ0e5+S/zPnvTK1Np\n6Te/kumK3vw+/yYYxzqUXdfLfeMtr+1/zY93nVLP//ltrfiNv+x2VJWH+3dlxOXtq/ZJReRzggyL\nfKBtqek2wNbSDdx9d6nJ8cCvSybM7AzgdeAn7j4/qCJTahkXlAyMY//14z+HPD6b/mx52WWfrWuf\na1sy/bllZVaOa50ytVGqbcnzl/c45W0P8axTwfNjFnfbss8f7zoXtjqDKzu1QETCF2RY5ACdzKwD\nsAUYBAwp3cDMWrn7tuhkf2B1dH5d4GXgOXf/a4A10qh+HcYO7RXkU4iIVHuBhYW7F5rZGGAOkAJk\nuftKM3sEyHX32cDdZtYfKAT2ABnR1W8BrgLONLOSeRnuviSoekVEpGLmVX0QOiCpqamem5sbdhki\nItWKmS1099RY7dRXgoiIxKSwEBGRmBQWIiISk8JCRERiUliIiEhMCgsREYkpaS6dNbMC4MNTeIjm\nwK5KKidMybIdoG1JVMmyLcmyHXBq23Kuu8fsKiFpwuJUmVluPNcaJ7pk2Q7QtiSqZNmWZNkOqJpt\n0WEoERGJSWEhIiIxKSw+My7sAipJsmwHaFsSVbJsS7JsB1TBtuichYiIxKQ9CxERianGhoWZPWFm\na8xsmZm9bGZNKmjXz8zWmlmemT1Q1XXGYmbfMrOVZlZsZhVeDWFmm8xsuZktMbOE7J73BLYloV8T\nADNrZmZvmtn66M+mFbQrir4mS8xsdlXXWZFYv2Mzq2dmM6LLs82sfdVXGZ84tiXDzApKvQ6jwqgz\nFjPLMrOdZraiguVmZn+KbucyM6vcgXrcvUbegC8DtaP3fw38upw2KUTG/j6PyDjgS4GLwq69TI0X\nAhcA7wCpx2m3CWgedr2nui3V4TWJ1vk48ED0/gPl/X1Flx0Iu9aT+R0DdwJ/id4fBMwIu+5T2JYM\n4Mmwa41jW64CegErKlh+PfA3IgNOpgPZlfn8NXbPwt3/4e6F0cn5RIZ9LSsNyHP3De5+FJgODKiq\nGuPh7qvdfW3YdVSGOLcl4V+TqAFExpQn+vOGEGs5UfH8jktv30zgS1Z2rN/EUF3+XmJy9/eIDBJX\nkQFERhd1jwxF3cTMWlXW89fYsCjjdiKJXFZrYHOp6fzovOrIgX+Y2UIzywy7mFNQXV6Tlh4dMjj6\n86wK2tU3s1wzm29miRIo8fyO/9Mm+qFrH3BmlVR3YuL9e7kpeuhmppm1rZrSKl2g/xtBjsEdOjN7\nCzi7nEU/dvdZ0TY/JjKs65TyHqKceVV++Vg82xGHvu6+1czOAt40szXRTypVqhK2JSFeEzj+tpzA\nw7SLvi7nAf80s+Xu/kHlVHjS4vkdJ8zrEEM8db4KTHP3I2b2bSJ7TNcEXlnlC/Q1SeqwcPdrj7fc\nzEYAXwe+5NGDfmXkA6U/ZbQBtlZehfGJtR1xPsbW6M+dZvYykd3zKg+LStiWhHhN4PjbYmY7zKyV\nu2+LHgrYWcFjlLwuG8zsHaAnkWPsYYrnd1zSJt/MagONOf4hkrDE3BZ3311qcjyRc5jVUaD/GzX2\nMJSZ9QN+BPR390MVNMsBOplZBzOrS+REXsJcsRIvM2tgZo1K7hM5uV/uFRXVQHV5TWYDI6L3RwCf\n22sys6ZmVi96vznQF1hVZRVWLJ7fcentuxn4ZwUfuMIWc1vKHNfvD6yuwvoq02xgePSqqHRgX8mh\n0EoR9hn+sG5AHpHje0uit5IrO84B3ijV7npgHZFPez8Ou+5ytuObRD5RHAF2AHPKbgeRK0GWRm8r\nE3E74t2W6vCaRGs8E3gbWB/92Sw6PxWYEL1/ObA8+rosB0aGXffxfsfAI0Q+XAHUB/4a/T9aAJwX\nds2nsC2/jP5fLAXmAl3CrrmC7ZgGbAOORf9PRgLfBr4dXW7A2Oh2Luc4V0eezE3f4BYRkZhq7GEo\nERGJn8JCRERiUliIiEhMCgsREYlJYSEiIjEpLEROgJkdOMX1Z0a/rY2ZNTSzp83sg2hvu++ZWR8z\nqxu9n9RfmpXqRWEhUkXMrCuQ4u4borMmEPnWcyd370qk99PmHunw7m1gYCiFipRDYSFyEqLfkn3C\nzFZExwkZGJ1fy8yeiu4pvGZmb5jZzdHVhhL9JreZdQT6AD9x92KIdPnh7q9H274SbS+SELSbK3Jy\nbgR6AJcAzYEcM3uPSJcd7YFuRHqaXQ1kRdfpS+RbuABdgSXuXlTB468AegdSuchJ0J6FyMm5gkhP\npUXuvgN4l8ib+xXAX9292N23E+k+okQroCCeB4+GyNGSPr1EwqawEDk5FQ30c7wBgD4l0qcSRPoi\nusTMjvc/WA84fBK1iVQ6hYXIyXkPGGhmKWbWgsiQlwuAfxMZSKeWmbUEri61zmrgfACPjFmRCzxc\nMsKcmXUyswHR+2cCBe5+rKo2SOR4FBYiJ+dlYBmRnkr/CdwfPez0IpEeQVcATwPZREaRA3id/w6P\nUUQGT8ozs+VExlIoGX/gi8AbwW6CSPzU66xIJTOzhu5+ILp3sIDIKIXbzew0Iucw+h7nxHbJY7wE\nPOhJMr66VH+6Gkqk8r1mZk2AusCj0T0O3P1TM/s5kXGRP6po5eggPa8oKCSRaM9CRERi0jkLERGJ\nSWEhIiIxKSxERCQmhYWIiMSksBARkZgUFiIiEtP/B+w8IkQsEoK7AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x535b15c0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "n_Cs = len(Cs)\n",
    "n_classes = 3\n",
    "scores =  np.zeros((n_classes,n_Cs))\n",
    "\n",
    "for j in range(n_classes):\n",
    "        scores[j][:] = np.mean(lr_cv_L2.scores_[j],axis = 0)\n",
    "    \n",
    "mse_mean = -np.mean(scores, axis = 0)\n",
    "pyplot.plot(np.log10(Cs), mse_mean.reshape(n_Cs,1)) \n",
    "#plt.plot(np.log10(reg.Cs)*np.ones(3), [0.28, 0.29, 0.30])\n",
    "pyplot.xlabel('log(C)')\n",
    "pyplot.ylabel('neg-logloss')\n",
    "pyplot.show()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 118,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "LogisticRegressionCV(Cs=[0.01, 0.1, 1, 10], class_weight=None, cv=5,\n",
       "           dual=False, fit_intercept=True, intercept_scaling=1.0,\n",
       "           max_iter=100, multi_class='ovr', n_jobs=1, penalty='l2',\n",
       "           random_state=None, refit=True, scoring='neg_log_loss',\n",
       "           solver='lbfgs', tol=0.0001, verbose=0)"
      ]
     },
     "execution_count": 118,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lrcv_L2 = LogisticRegressionCV(Cs=Cs, cv = 5, scoring='neg_log_loss', penalty='l2', multi_class='ovr')\n",
    "lrcv_L2.fit(X_train, y_train)    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 120,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{0: array([[-0.26756356, -0.28897583, -0.32972302, -0.36144027],\n",
       "        [-0.26722783, -0.28540003, -0.33102239, -0.38122333],\n",
       "        [-0.25100259, -0.24697348, -0.25377791, -0.26260233],\n",
       "        [-0.25730585, -0.26204053, -0.27962417, -0.29311477],\n",
       "        [-0.24579925, -0.25438646, -0.2734083 , -0.2932277 ]]),\n",
       " 1: array([[-0.52531684, -0.5442236 , -0.58712464, -0.64150955],\n",
       "        [-0.54611915, -0.60962367, -0.72336696, -0.87468135],\n",
       "        [-0.51639971, -0.53402067, -0.57937013, -0.63968468],\n",
       "        [-0.53230443, -0.55011538, -0.58528926, -0.6282376 ],\n",
       "        [-0.52173402, -0.56514259, -0.64786627, -0.75624216]]),\n",
       " 2: array([[-0.55456753, -0.57799731, -0.62863891, -0.68291076],\n",
       "        [-0.55416204, -0.58387447, -0.66817139, -0.78909117],\n",
       "        [-0.51331459, -0.49372532, -0.51593168, -0.55076886],\n",
       "        [-0.53507005, -0.52877337, -0.54096511, -0.56042541],\n",
       "        [-0.5376813 , -0.5655481 , -0.64992104, -0.75608838]])}"
      ]
     },
     "execution_count": 120,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lrcv_L2.scores_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 122,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYsAAAEKCAYAAADjDHn2AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4wLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvpW3flQAAIABJREFUeJzt3Xl8VOXZ//HPRdgXWQybLAYhiCCy\nGDYBbbW22MeCCxXcERTxeRCf2talT7V16aK22kVaxQVFCqhoFZUWlypqZUnCvgvIEhYJ+w5Zrt8f\nM/BLY5IZIJMzM/m+X6+8mHPmnJnrZEi+Ofe5z32buyMiIlKWKkEXICIi8U9hISIiESksREQkIoWF\niIhEpLAQEZGIFBYiIhKRwkJERCJSWIiISEQKCxERiahq0AWUl9TUVE9LSwu6DBGRhJKdnb3d3RtH\n2i5pwiItLY2srKygyxARSShmtj6a7dQMJSIiESksREQkIoWFiIhEpLAQEZGIFBYiIhKRwkJERCJS\nWIiISEQKCxGRBPbBsq95Izsn5u+jsBARSVBvZOcwamI2k+ZuoKDQY/peCgsRkQT0wudf8ePXF9Ln\nrNOZMLwnKVUspu+XNMN9iIhUBu7Okx+s4s//Ws1l5zbjD0O7UqNqSszfV2EhIpIgCgudB6ctYeLs\nDQzt0YpfXdk55mcUx8S0GcrMBpjZSjNbbWb3lfD8MDPLNbMF4a9bizzX2szeN7PlZrbMzNJiWauI\nSDw7ml/IXa8uYOLsDYy6qC2/uariggJieGZhZinAWOBSIAfINLNp7r6s2KavuvvoEl5iAvArd//A\nzOoChbGqVUQknh06WsCoidnMXJXLfZd1YNRFbSu8hlg2Q/UEVrv7WgAzmwIMAoqHxTeYWUegqrt/\nAODu+2NYp4hI3NpzMI/hL2cyf8MufntVZ4b2bB1IHbFshmoBbCyynBNeV9zVZrbIzKaaWavwuvbA\nbjN708zmm9kT4TMVEZFKY9vewwwZN4vFOXsYe133wIICYhsWJTWmFe8I/A6Q5u7nAR8CL4fXVwX6\nAz8BegBnAcO+8QZmI80sy8yycnNzy6tuEZHAbdhxkMHPzGLDzoO8OKwHl3VuHmg9sQyLHKBVkeWW\nwOaiG7j7Dnc/El58Dji/yL7z3X2tu+cDbwHdi7+Bu49z9wx3z2jcOOKsgCIiCWHF1r0MfuYL9h7O\nY9JtvemXnhp0STENi0wg3czamFl1YCgwregGZlY0KgcCy4vs29DMjiXAxURxrUNEJNFlr9/FNc/M\noooZr9/eh66tGgRdEhDDC9zunm9mo4EZQArworsvNbOHgSx3nwaMMbOBQD6wk3BTk7sXmNlPgI/M\nzIBsQmceIiJJa+aqXEa9kk2z+jWZMLwnrRrVDrqk48w9tuOJVJSMjAzPysoKugwRkZPyzsLN3P3a\nAtKb1OPl4T1pXK9GhbyvmWW7e0ak7XQHt4hIwP42Zz0/f2sJPc5sxPPDMjitZrWgS/oGhYWISEDc\nnb98soYnZqzkkg5NGHt9d2pWi8+7BBQWIiIBcHd+9d5ynv/8K67s1oLHB59HtZT4HQhcYSEiUsHy\nCwq5783FTM3OYdgFaTx4eUeqVOA4TydDYSEiUoEO5xUwZvJ83l/2NT/6TnvGXNKOUKfP+KawEBGp\nIPsO5zFyQjaz1u7goYGduPmCtKBLiprCQkSkAuzYf4Rh4zNZvmUvfxjSlSu6lTRUXvxSWIiIxNjm\n3Ye44YU5bNp1iHE3nc/FHZoGXdIJU1iIiMTQmtz93Pj8HPYdzueVEb3o2aZR0CWdFIWFiEiMLM7Z\nw83j51LFYMrtvel0Rv2gSzppCgsRkRiYtWYHt03Ion6taky8tRdtUusEXdIpUViIiJSz95duZfTk\n+ZzZqDavjOhFs/o1gy7plCksRETK0dTsHO59YxGdW9Rn/LAeNKxTPeiSyoXCQkSknDz/2VoefW85\n/dql8uyN51OnRvL8ik2eIxERCYi78/v3V/H0x6v5fudmPDWkKzWqxueAgCdLYSEicgoKCp1fTFvC\nxNkbGNqjFb+6sjMpcT7O08lQWIiInKSj+YXc/doC3l20hVEXteXeAWcnxDhPJ0NhISJyEg4ezeeO\nifOYuSqX+y/rwO0XtQ26pJhSWIiInKA9B/MY/nIm8zfs4rGrOzOkR+ugS4o5hYWIyAnYtvcwN704\nl7W5B/jL9d0ZcG7zoEuqEAoLEZEobdhxkBtemMP2/Ud4cVgP+qWnBl1ShVFYiIhEYcXWvdz4wlzy\nCgqZdFtvurZqEHRJFSp+J3wVEYkT2et3cs0zs0gx4/Xb+1S6oIAYh4WZDTCzlWa22szuK+H5YWaW\na2YLwl+3Fnv+NDPbZGZPx7JOEZHSzFyVyw3Pz+X0ujWYekcf0pvWC7qkQMSsGcrMUoCxwKVADpBp\nZtPcfVmxTV9199GlvMwjwMxY1SgiUpZ3Fm7m7tcWkN6kHi8P70njejWCLikwsTyz6Amsdve17n4U\nmAIMinZnMzsfaAq8H6P6RERKNXH2esZMmU+3Vg2ZcnvvSh0UENuwaAFsLLKcE15X3NVmtsjMpppZ\nKwAzqwL8HvhpDOsTEfkGd2fsx6v5+VtLuPjsJkwY0ZPTalYLuqzAxTIsSrrn3YstvwOkuft5wIfA\ny+H1/w1Md/eNlMHMRppZlpll5ebmnnLBIlK5FRY6v3pvOU/MWMmV3VrwzI3nU7Nacg0IeLJi2XU2\nB2hVZLklsLnoBu6+o8jic8Bj4cd9gP5m9t9AXaC6me139/uK7T8OGAeQkZFRPIhERKKWX1DIfW8u\nZmp2DsMuSOPByztSJQkHBDxZsQyLTCDdzNoAm4ChwHVFNzCz5u6+Jbw4EFgO4O7XF9lmGJBRPChE\nRMrL4bwC7pw8nw+Wfc2PvtOeMZe0S9oBAU9WzMLC3fPNbDQwA0gBXnT3pWb2MJDl7tOAMWY2EMgH\ndgLDYlWPiEhJ9h3OY+SEbGat3cFDAztx8wVpQZcUl8w9OVpvMjIyPCsrK+gyRCSB7Nh/hGHjM1m+\nZS+/v6YLg7qW1AcnuZlZtrtnRNpOw32ISKW0efchbnhhDpt2HWLcTedzcYemQZcU1xQWIlLprN62\nn5temMO+I/lMvLUXPdIaBV1S3FNYiEilsihnN8PGZ1LFjCkje9PpjPpBl5QQFBYiUml8sWY7t72c\nRcM61Zk4ohdpqXWCLilhKCxEpFKYsXQrd06eT9rptZkwvBfN6tcMuqSEorAQkaT3etZG7n1jEee1\nbMBLt/SgQe3qQZeUcBQWIpLUnv9sLY++t5z+6ak8c8P51KmhX3snQ981EUlK7s7v31/F0x+v5vud\nm/HUkK7UqKpxnk6WwkJEkk5BofPg20v425wNXNuzFY9e0ZkUjfN0ShQWIpJUjuYXcvdrC3h30Rbu\n+FZb7vne2RrnqRwoLEQkaRw8ms+oifP4dFUu91/Wgdsvaht0SUlDYSEiSWHPwTxueWkuCzbu5rGr\nOzOkR+ugS0oqCgsRSXjb9h7mxhfm8tX2A/zl+u4MOLd50CUlHYWFiCS09TsOcMMLc9ix/yjjb+lB\n33apQZeUlBQWIpKwlm/Zy00vziW/oJBJt/Wma6sGQZeUtBQWIpKQstfv5JbxmdSuXpVJt/chvWm9\noEtKagoLEUk4n6zcxqiJ2TSvX4tXRvSkZcPaQZeU9BQWIpJQpi3czN2vLqB903pMGNGT1Lo1gi6p\nUlBYiEjCeGX2eh58ewk90hrx/M0ZnFazWtAlVRoKCxGJe+7O2I9X87v3V3FJhyaMvb47NatpnKeK\npLAQkbhWWOj8avpyXvj8K67s1oLHB59HtZQqQZdV6SgsRCRu5RcUcu8bi3ljXg7DLkjjwcs7UkUD\nAgZCYSEicelwXgF3Tp7PB8u+5u5L23Pnxe00IGCAYnouZ2YDzGylma02s/tKeH6YmeWa2YLw163h\n9V3NbJaZLTWzRWY2JJZ1ikh82Xc4j2Hj5/LBsq95eFAnxlySrqAIWMzOLMwsBRgLXArkAJlmNs3d\nlxXb9FV3H11s3UHgJnf/0szOALLNbIa7745VvSISH3bsP8Kw8Zks37KXPw7tyqCuLYIuSYhtM1RP\nYLW7rwUwsynAIKB4WHyDu68q8nizmW0DGgMKC5Ektmn3IW58YQ6bdh3iuZsy+HaHJkGXJGGxbIZq\nAWwsspwTXlfc1eGmpqlm1qr4k2bWE6gOrIlNmSISD1Zv28/gv35B7r4jTLy1l4IizsQyLEpqYPRi\ny+8Aae5+HvAh8PJ/vIBZc+AV4BZ3L/zGG5iNNLMsM8vKzc0tp7JFpKItytnNNc/OIq/AeXVkH3qk\nNQq6JCkmqrAws75mVif8+AYze9LMzoywWw5Q9EyhJbC56AbuvsPdj4QXnwPOL/KepwHvAT9399kl\nvYG7j3P3DHfPaNy4cTSHIiJx5os127l23GxqV09h6qg+dDzjtKBLkhJEe2bxV+CgmXUB7gHWAxMi\n7JMJpJtZGzOrDgwFphXdIHzmcMxAYHl4fXXg78AEd389yhpFJMHMWLqVYeMzadGwFlNHXUBaap2g\nS5JSRBsW+e7uhC5Q/9Hd/wiUOR6wu+cDo4EZhELgNXdfamYPm9nA8GZjwt1jFwJjgGHh9dcAFwLD\ninSr7XpCRyYice31rI3cMTGbjs1P47Xb+9Csfs2gS5IyWCgDImxkNhP4J3ALoV/iucACd+8c2/Ki\nl5GR4VlZWUGXISJReP6ztTz63nL6p6fyzA3nU6eG7g8Oipllu3tGpO2iPbMYAhwBRrj7VkK9mp44\nhfpEpBJyd56YsYJH31vOf3VuzvM3ZygoEkS0n9I+Qs1PBWbWHugATI5dWSKSbAoKnQfeXsKkORu4\ntmcrHr2iMyka5ylhRHtm8SlQw8xaAB8Rao56KVZFiUhyOZpfyF1T5jNpzgbu+FZbfn2lgiLRRBsW\n5u4HgauAP7v7lUCn2JUlIsni4NF8bp2QxbuLtvCz73fg3gEdNM5TAoq2GcrMrA9wPTAivE4zj4hI\nmXYfPMrwlzJZsHE3j199Htf0+MYgDZIgog2L/wXuB/4e7v56FvBx7MoSkUS3be9hbnxhLl9tP8Bf\nru/OgHObR95J4lZUYeHuM4GZZlbPzOqGBwccE9vSRCRRrd9xgBtemMPO/UcZf0sP+rZLDbokOUXR\nDvfR2czmA0uAZWaWbWa6ZiEi37B8y14GPzOL/YfzmXRbbwVFkoj2AvezwN3ufqa7twZ+TGgsJxGR\n47LW7WTIs7NIMeP1UX3o0qpB0CVJOYn2mkUddz9+jcLdPzk2sKCICMAnK7cxamI2zevX4pURPWnZ\nsHbQJUk5ijYs1prZA4SGCwe4AfgqNiWJSKKZtnAzd7+6gLOb1ePl4T1JrVsj6JKknEXbDDWc0Ex1\nbxIaDbYxoRvzRKSSe2X2eu6aMp/uZzZk8sjeCookFW1vqF2o95OIFOHuPP2v1fz+g1V855wmPH1d\nd2pW0+1XyarMsDCzd/jm7HbHufvA0p4TkeRVWOg8+t5yXvz3V1zVrQWPDT6PaimxnHhTghbpzOJ3\nFVKFiCSM/IJC7n1jMW/My+GWvmk88F8dqaJxnpJemWERvhlPRASAw3kFjJ40nw+Xf83dl7bnzovb\naZynSiKqaxZmtphvNkftAbKAR919R3kXJiLxZdeBo9zxt2zmfLWThwd14qY+aUGXJBUo2q6z/wAK\ngEnh5aGAEQqMl4AflHtlIhI3MtftZMzk+ezYf5Q/DOnKoK4tgi5JKli0YdHX3fsWWV5sZv92975m\ndkMsChOR4BUWOn+duYYnP1hFq4a1ePO/L+DcFvWDLksCEG1Y1DWzXu4+B8DMegJ1w8/lx6QyEQlU\n7r4j3P3aAj77cjs/6HIGv77yXOrVrBZ0WRKQaMPiVuBFM6tLqPlpLzAiPOTHb2JVnIgE49+rt3PX\nlAXsO5zHb6/qzJAerXQhu5KL9qa8TKCzmdUnNGve7iJPvxaTykSkwuUXFPKnj77kzx+vpm3juvzt\n1l6c3axe0GVJHIi2N1R94BfAheHlmcDD7r4nhrWJSAXauucwY6bMZ+5XO/nh+S15aFAnalePtvFB\nkl20/xNeJDSXxTXh5RuB8YTm5BaRBPfxim38+PWFHM4r4MlrunBV95ZBlyRxJtr789u6+y/cfW34\n6yHgrEg7mdkAM1tpZqvN7L4Snh9mZrlmtiD8dWuR5242sy/DXzdHf0giEq28gkJ+M305t7yUSZN6\nNXjnzn4KCilRtGcWh8ysn7t/DmBmfYFDZe1gZinAWOBSIAfINLNp7r6s2KavuvvoYvs2ItTslUHo\nZsDs8L67oqxXRCLI2XWQOyfPZ/6G3VzfqzUPXN5RAwFKqaINizuAl49d4AZ2AsMi7NMTWB2erxsz\nmwIMAoqHRUm+B3zg7jvD+34ADAAmR1mviJRhxtKt/PT1hbjD09d14/Lzzgi6JIlz0faGWgB0MbPT\nwst7o9itBbCxyHIO0KuE7a42swuBVcCP3H1jKft+45ZRMxsJjARo3bp1FCWJVG5H8gv4zfQVvPTF\nOjq3qM/T13XjzNM16aVEFmmI8rtLWQ+Auz9Z1u4lrCs+vtQ7wGR3P2Jmo4CXgYuj3Bd3HweMA8jI\nyCh1KHURgXXbDzB68jyWbNrL8L5tuPeys6lRVc1OEp1IZxan0sE6B2hVZLklsLnoBsUGIHwOeKzI\nvt8qtu8np1CLSKX2zsLN3P/mYlKqGM/dlMGlHZsGXZIkmEhDlD90Cq+dCaSbWRtgE6HBB68ruoGZ\nNXf3LeHFgcDy8OMZwK/NrGF4+bvA/adQi0ildDivgIfeWcbkuRvo3roBf76uOy0a1Aq6LElAJ3zH\njZnNc/fukbZz93wzG03oF38K8KK7LzWzh4Esd58GjDGzgYTGlzp+0dzdd5rZI4QCB0I3AO480VpF\nKrPV2/YxetJ8Vmzdxx3fasvdl7bXbHZy0sz9xJr6zWy+u3eLUT0nLSMjw7OysoIuQyQuTM3O4YG3\nllC7egq/v6YL3zq7SdAlSZwys2x3z4i03cncy//eSewjIhXgwJF8Hnh7CW/O20Tvsxrxx6HdaHpa\nzaDLkiRwwmHh7j+PRSEicmqWb9nL6EnzWLv9AHddks6YS9JJ0dzYUk6iHUhwH6VPq/rjYzfeiUjF\nc3cmz93IQ+8s5bRa1fjbiF5c0C416LIkyUR7ZvEkoW6vkwjdAzEUaAasJDTI4LdiUZyIlG3f4Tzu\nf3Mx7y7aQv/0VJ4a0pXUujWCLkuSULRhMcDdi959Pc7MZrv7w2b2s1gUJiJlW5yzh9GT55Gz6xD3\nDDibURe2pYqanSRGog2LQjO7BpgaXh5c5DndOS1Sgdydl75Yx6+nLye1bg1eHdmbjLRGQZclSS7a\nsLge+CPwF0LhMBu4wcxqAaPL2lFEys/ug0e5Z+oi3l/2Nd85pwlPDO5CwzrVgy5LKoFoBxJcC/yg\nlKc/L79yRKQ02et3MWbyfLbtO8wDl3dkeN80zYstFSaq2znNrL2ZfWRmS8LL55mZutCKVIDCQueZ\nmWu45tlZVKkCU0ddwIh+bRQUUqGivff/OUJjM+UBuPsiQj2iRCSGduw/wvCXM/ntP1bwvU5NeW9M\nf7q0ahB0WVIJRXvNora7zy32l0x+DOoRkbDZa3dw15T57DqYxyNXnMsNvVrrbEICE21YbDeztoR7\nPpnZYGBL2buIyMkoKHTGfryaP3y4irTT6/DisB50OqN+0GVJJRdtWPwPoUmGOpjZJuArQj2kRKQc\nbdt7mP99dQFfrNnBFV3P4NErO1O3xskM4SZSvqL9X7gJGA98DDQC9gI3Aw/HqC6RSuezL3P50asL\n2H8kn8cHn8cPz2+pZieJG9GGxdvAbmAexWa7E5FTk19QyFMfruIvn6whvUldJt/Wm/SmpzJJpUj5\nizYsWrr7gJhWIlIJbd59iLumzCdz3S6G9mjFL37QiVrVNS+2xJ9ow+ILM+vs7otjWo1IJfLR8q/5\n8esLycsv5I9DuzKoa4ugSxIpVbRh0Q8YZmZfAUcIjTzr7n5ezCoTSVJH8wt5/J8reP7zr+jY/DTG\nXt+dNql1gi5LpEzRhsVlMa1CpJLYuPMgoyfPZ+HG3dzU50x+9v1zqFlNzU4S/6IdG2p9rAsRSXb/\nWLyFe95YBMBfr+/OZZ2bB1yRSPTUgVskxg7nFfDr6cuZMGs9XVo14Olru9GqUe2gyxI5IQoLkRha\nm7uf0ZPms2zLXm7r34affq8D1atGOySbSPxQWIjEyNsLNvGzNxdTrWoVXrg5g0vOaRp0SSInLaZ/\n4pjZADNbaWarzey+MrYbbGZuZhnh5Wpm9rKZLTaz5WZ2fyzrFClPh44WcO/URdw1ZQEdzziN6WP6\nKygk4cXszMLMUoCxwKVADpBpZtPcfVmx7eoBY4A5RVb/EKjh7p3NrDawzMwmu/u6WNUrUh5Wfb2P\n0ZPm8eW2/Yz+djv+9zvpVE1Rs5Mkvlj+L+4JrHb3te5+FJgCDCphu0eAx4HDRdY5UMfMqgK1gKOE\nxqMSiUvuzmuZGxn49OfsPHCUCcN78pPvna2gkKQRy//JLYCNRZZzwuuOM7NuQCt3f7fYvlOBA4SG\nQd8A/M7ddxZ/AzMbaWZZZpaVm5tbrsWLRGv/kXx+9OoC7nljEd1bN2T6mP70T28cdFki5SqWF7hL\nGi7Tjz9pVgV4ChhWwnY9gQLgDKAh8JmZfRieC/z/v5j7OEJDp5ORkeHfeBWRGFu6eQ93TprPuh0H\nuPvS9vzPt9uRUkUjxUryiWVY5ACtiiy35D9HrK0HnAt8Eh6GuRkwzcwGAtcB/3T3PGCbmf0byAD+\nIyxEguLuTJyzgUfeXUbD2tWYdFtvep91etBlicRMLJuhMoF0M2tjZtUJzdk97diT7r7H3VPdPc3d\n04DZwEB3zyLU9HSxhdQBegMrYlirSNT2HMrjfybN44G3lnBB29OZPqa/gkKSXszOLNw938xGAzOA\nFOBFd19qZg8DWe4+rYzdxxKabGkJoeas8e6+KFa1ikRr4cbdjJ48jy27D3P/ZR24rf9ZVFGzk1QC\n5p4cTf0ZGRmelZUVdBmSpNydFz7/isf+uYIm9Wryp2u7cf6ZDYMuS+SUmVm2u2dE2k53cItEsOvA\nUX46dSEfLt/Gdzs25fHB59GgdvWgyxKpUAoLkTJkrdvJnZPns2P/UX75g47cfEGa5sWWSklhIVKC\nwkLnrzPX8OQHq2jZsBZv3HEBnVvWD7oskcAoLESK2b7/CD96dQGffbmdy89rzm+u6ky9mtWCLksk\nUAoLkSK+WLOdu6YsYO+hPH59ZWeu7dlKzU4iKCxEACgodP700Zf86V9fclZqHSYM78k5zU8LuiyR\nuKGwkErv672HuWvKfGav3cnV3Vvy8KBO1KmhHw2RovQTIZXaJyu3cfdrCzl0tIDf/bALg89vGXRJ\nInFJYSGVUl5BIb9/fxXPzFxDh2b1ePq6brRrUi/oskTilsJCKp1Nuw9x56R5zNuwm+t6tebByztS\ns1pK0GWJxDWFhVQq7y/dyk+nLqKg0Pnztd34QZczgi5JJCEoLKRSOJJfwG//sYLx/15H5xb1+fO1\n3UhLrRN0WSIJQ2EhSW/9jgOMnjSfxZv2MOyCNO7/fgdqVFWzk8iJUFhIUnt30Wbuf2MxZvDsjefz\nvU7Ngi5JJCEpLCQpHc4r4JF3l/G3ORvo1roBf762Gy0b1g66LJGEpbCQpLN6235GT5rHiq37uP2i\ns/jJd8+mWkosJ4UUSX4KC0kqb87L4edvLaFmtRTG39KDb5/dJOiSRJKCwkKSwsGj+Tz49lKmZufQ\ns00j/jS0G83q1wy6LJGkobCQhLdi615GT5rPmtz9jLkknTEXt6Oqmp1EypXCQhKWuzMlcyO/nLaU\n02pVY+KIXvRtlxp0WSJJSWEhCWnf4Tx+9vclvLNwM/3apfLUkK40rlcj6LJEkpbCQhLOkk17GD1p\nHht2HuSn3zubOy5qS5UqmqBIJJYUFpIw3J0Js9bzq/eW06hOdaaM7EPPNo2CLkukUojpVUAzG2Bm\nK81stZndV8Z2g83MzSyjyLrzzGyWmS01s8Vmpq4tldieg3mMmpjNL6YtpV96KtPv6q+gEKlAMTuz\nMLMUYCxwKZADZJrZNHdfVmy7esAYYE6RdVWBicCN7r7QzE4H8mJVq8S3eRt2ceek+Xy99zA//69z\nGNGvjebFFqlgsTyz6Amsdve17n4UmAIMKmG7R4DHgcNF1n0XWOTuCwHcfYe7F8SwVolDhYXOuE/X\ncM0zszCD10f14db+ZykoRAIQy7BoAWwsspwTXnecmXUDWrn7u8X2bQ+4mc0ws3lmdk8M65Q4tHn3\nIW6dkMWvp6/gO+c05b0x/enWumHQZYlUWrG8wF3Sn39+/EmzKsBTwLAStqsK9AN6AAeBj8ws290/\n+o83MBsJjARo3bp1+VQtgTiSX0D2ul3MXJXLzFW5rNi6j+opVXhkUCdu6H2mziZEAhbLsMgBWhVZ\nbglsLrJcDzgX+CT8i6AZMM3MBob3nenu2wHMbDrQHfiPsHD3ccA4gIyMDEcSyrrtB/j0y1xmrszl\nizU7OJRXQLUUo0daI+67rAMDOjXTBEUicSKWYZEJpJtZG2ATMBS47tiT7r4HOH67rZl9AvzE3bPM\nbA1wj5nVBo4CFxE6C5EEduBIPrPW7AgFxKpc1u84CEDrRrX5YUZLLkxvTJ+2p1Onhnp0i8SbmP1U\nunu+mY0GZgApwIvuvtTMHgay3H1aGfvuMrMnCQWOA9Pd/b1Y1Sqx4e6s2LqPmaty+XRVLpnrdpJX\n4NSqlsIFbU9nRL82XJjeWGcPIgnA3JOj9SYjI8OzsrKCLqPS233wKJ99uf14QGzbdwSADs3qcVH7\nxlzUvjHnpzXUtKYicSJ8PTgj0nY635dTUlDoLMzZzcyVoaalRTm7KXSoX6sa/dJTuah9Yy5Mb6zh\nwkUSnMJCTtjXew8f77X0+Zfb2XMoDzPo0rIBd16czkVnN6ZLywakaLwmkaShsJCISurWCtCkXg0u\n7diUi9o3pl+7VBrWqR5wpSISKwoLKVGkbq0XtW9Mh2b1dP+DSCWhsBBA3VpFpGz6ya+k1K1VRE6E\nwqISKatb6/C+bdStVURKpbD/SO0aAAAIlElEQVRIYurWKiLlRWGRZNStVURiQWGR4NStVUQqgsIi\nAa3bfuD4dQd1axWRiqCwSADHurXOXJXLp1+qW6uIVDz9dolD6tYqIvFGYREndh04yuer1a1VROKT\nwiIg6tYqIolEYVGB1K1VRBKVwiKGjuQXkLVuF5+qW6uIJDiFRTlTt1YRSUYKi1Okbq0iUhnoN9gJ\nKtqtdebKXLLWq1uriCQ/hUUU1K1VRCo7hUUJ1K1VROQ/KSzC1K1VRKR0MQ0LMxsA/BFIAZ5399+W\nst1g4HWgh7tnFVnfGlgG/NLdfxeLGjftPsSIlzLVrVVEpAwxCwszSwHGApcCOUCmmU1z92XFtqsH\njAHmlPAyTwH/iFWNAE3r1aBFg1pc0a2FurWKiJQilmcWPYHV7r4WwMymAIMInSkU9QjwOPCToivN\n7ApgLXAghjVSNaUKLwzrEcu3EBFJeFVi+NotgI1FlnPC644zs25AK3d/t9j6OsC9wEMxrE9ERKIU\ny7AoqS3Hjz9pVoVQM9OPS9juIeApd99f5huYjTSzLDPLys3NPaViRUSkdLFshsoBWhVZbglsLrJc\nDzgX+CR8jaAZMM3MBgK9gMFm9jjQACg0s8Pu/nTRN3D3ccA4gIyMDEdERGIilmGRCaSbWRtgEzAU\nuO7Yk+6+B0g9tmxmnwA/CfeG6l9k/S+B/cWDQkREKk7MmqHcPR8YDcwAlgOvuftSM3s4fPYgIiIJ\nwtyTo/UmIyPDs7KyIm8oIiLHmVm2u2dE2i6WF7hFRCRJKCxERCSipGmGMrNcYP0pvEQqsL2cyglS\nshwH6FjiVbIcS7IcB5zasZzp7o0jbZQ0YXGqzCwrmna7eJcsxwE6lniVLMeSLMcBFXMsaoYSEZGI\nFBYiIhKRwuL/Gxd0AeUkWY4DdCzxKlmOJVmOAyrgWHTNQkREItKZhYiIRFRpw8LMnjCzFWa2yMz+\nbmYNStlugJmtNLPVZnZfRdcZiZn90MyWmlmhmZXaG8LM1pnZYjNbYGZxeav7CRxLXH8mAGbWyMw+\nMLMvw/82LGW7gvBnssDMplV0naWJ9D02sxpm9mr4+TlmllbxVUYnimMZZma5RT6HW4OoMxIze9HM\ntpnZklKeNzP7U/g4F5lZ93ItwN0r5RfwXaBq+PFjwGMlbJMCrAHOAqoDC4GOQdderMZzgLOBT4CM\nMrZbB6QGXe+pHksifCbhOh8H7gs/vq+k/1/h5/YHXevJfI+B/waeCT8eCrwadN2ncCzDgKeDrjWK\nY7kQ6A4sKeX57xOaWdSA3sCc8nz/Sntm4e7ve2iwQ4DZhIZQL+74bH/ufhQ4Nttf3HD35e6+Mug6\nykOUxxL3n0nYIODl8OOXgSsCrOVERfM9Lnp8U4FLLD7nI06U/y8RufunwM4yNhkETPCQ2UADM2te\nXu9facOimOGUPNd3xNn+EogD75tZtpmNDLqYU5Aon0lTd98CEP63SSnb1QxP4DU7PJVwPIjme3x8\nm/AfXXuA0yukuhMT7f+Xq8NNN1PNrFUJzyeCmP5sxHI+i8CZ2YeEJlUq7v/c/e3wNv8H5AN/K+kl\nSlhX4d3HojmOKPR1981m1gT4wMxWhP9SqVDlcCxx8ZlA2cdyAi/TOvy5nAX8y8wWu/ua8qnwpEXz\nPY6bzyGCaOp8B5js7kfMbBShM6aLY15Z+YvpZ5LUYeHu3ynreTO7GbgcuMTDjX7FRJrtr0JEOo4o\nX2Nz+N9tZvZ3QqfnFR4W5XAscfGZQNnHYmZfm1lzd98SbgrYVsprHPtc1oYnAOtGqI09SNF8j49t\nk2NmVYH6lN1EEpSIx+LuO4osPkfoGmYiiunPRqVthjKzAcC9wEB3P1jKZsdn+zOz6oQu5MVNj5Vo\nmVkdM6t37DGhi/sl9qhIAInymUwDbg4/vhn4xlmTmTU0sxrhx6lAX2BZhVVYumi+x0WPbzDwr1L+\n4ApaxGMp1q4/kNBkbYloGnBTuFdUb2DPsabQchH0Ff6gvoDVhNr3FoS/jvXsOAOYXmS77wOrCP21\n939B113CcVxJ6C+KI8DXwIzix0GoJ8jC8NfSeDyOaI8lET6TcI2nAx8BX4b/bRRenwE8H358AbA4\n/LksBkYEXXdZ32PgYUJ/XAHUBF4P/xzNBc4KuuZTOJbfhH8uFgIfAx2CrrmU45gMbAHywj8nI4BR\nwKjw8waMDR/nYsroHXkyX7qDW0REIqq0zVAiIhI9hYWIiESksBARkYgUFiIiEpHCQkREIlJYiJwA\nM9t/ivtPDd+tjZnVNbNnzWxNeLTdT82sl5lVDz9O6ptmJbEoLEQqiJl1AlLcfW141fOE7npOd/dO\nhEY/TfXQgHcfAUMCKVSkBAoLkZMQvkv2CTNbEp4nZEh4fRUz+0v4TOFdM5tuZoPDu11P+E5uM2sL\n9AJ+7u6FEBryw93fC2/7Vnh7kbig01yRk3MV0BXoAqQCmWb2KaEhO9KAzoRGml0OvBjepy+hu3AB\nOgEL3L2glNdfAvSISeUiJ0FnFiInpx+hkUoL3P1rYCahX+79gNfdvdDdtxIaPuKY5kBuNC8eDpGj\nx8b0EgmawkLk5JQ20U9ZEwAdIjSmEoTGIupiZmX9DNYADp9EbSLlTmEhcnI+BYaYWYqZNSY05eVc\n4HNCE+lUMbOmwLeK7LMcaAfgoTkrsoCHjs0wZ2bpZjYo/Ph0INfd8yrqgETKorAQOTl/BxYRGqn0\nX8A94WanNwiNCLoEeBaYQ2gWOYD3+M/wuJXQ5EmrzWwxobkUjs0/8G1gemwPQSR6GnVWpJyZWV13\n3x8+O5hLaJbCrWZWi9A1jL5lXNg+9hpvAvd7ksyvLolPvaFEyt+7ZtYAqA48Ej7jwN0PmdkvCM2L\nvKG0ncOT9LyloJB4ojMLERGJSNcsREQkIoWFiIhEpLAQEZGIFBYiIhKRwkJERCJSWIiISET/D1C+\nzUotfm2VAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x4ae46a90>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "n_Cs = len(Cs)\n",
    "n_classes = 3\n",
    "scores =  np.zeros((n_classes,n_Cs))\n",
    "\n",
    "for j in range(n_classes):\n",
    "        scores[j][:] = np.mean(lrcv_L2.scores_[j],axis = 0)\n",
    "    \n",
    "mse_mean = -np.mean(scores, axis = 0)\n",
    "pyplot.plot(np.log10(Cs), mse_mean.reshape(n_Cs,1)) \n",
    "#plt.plot(np.log10(reg.Cs)*np.ones(3), [0.28, 0.29, 0.30])\n",
    "pyplot.xlabel('log(C)')\n",
    "pyplot.ylabel('neg-logloss')\n",
    "pyplot.show()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.14"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
